RuntimeError: quantile() input tensor is too large

[noivan0]

Traceback (most recent call last):
File "D:\code\git-DS\EfficientAD\efficientad.py", line 451, in
main()
File "D:\code\git-DS\EfficientAD\efficientad.py", line 268, in main
q_st_start, q_st_end, q_ae_start, q_ae_end = map_normalization(
File "C:\Users\2878045\AppData\Roaming\Python\Python39\site-packages\torch\autograd\grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "D:\code\git-DS\EfficientAD\efficientad.py", line 374, in map_normalization
q_st_start = torch.quantile(maps_st, q=0.9)
RuntimeError: quantile() input tensor is too large

anyone solve this?

[FelixFu520]

Traceback (most recent call last): File "D:\code\git-DS\EfficientAD\efficientad.py", line 451, in main() File "D:\code\git-DS\EfficientAD\efficientad.py", line 268, in main q_st_start, q_st_end, q_ae_start, q_ae_end = map_normalization( File "C:\Users\2878045\AppData\Roaming\Python\Python39\site-packages\torch\autograd\grad_mode.py", line 27, in decorate_context return func(*args, **kwargs) File "D:\code\git-DS\EfficientAD\efficientad.py", line 374, in map_normalization q_st_start = torch.quantile(maps_st, q=0.9) RuntimeError: quantile() input tensor is too large

anyone solve this?

torch.quantile() limited process 16 million elements only，you could break maps_st into pieces, and process, like:
_q_st_start = []
for i in range(n):
_q_st_start,append(torch.quantile(maps_st[i: i+k], q=0.9))
q_st_start = torch.stack(_q_st_start).mean()

[Dirk10000]

Perhaps too late, use Histograms:

`@torch.no_grad()
def map_normalization_fast(validation_loader, teacher, student, autoencoder,
teacher_mean, teacher_std, desc='Map normalization'):

# Histogram with 10000 entries
hist_st_sum= torch.zeros(10000)
hist_ae_sum= torch.zeros(10000)
allcount= 0

# ignore augmented ae image
for image, _ in tqdm(validation_loader, desc=desc):
    if on_gpu:
        image = image.cuda()
        hist_st_sum= hist_st_sum.cuda()
        hist_ae_sum= hist_ae_sum.cuda()
                    
    map_combined, map_st, map_ae = predict(
        image=image, teacher=teacher, student=student,
        autoencoder=autoencoder, teacher_mean=teacher_mean,
        teacher_std=teacher_std)

    allcount= allcount + (map_st.size()[0]*map_st.size()[1]*map_st.size()[2]*map_st.size()[3])
    
    hist_st= torch.histc( map_st, 10000, 0.0, 10.0)
    hist_st_sum= hist_st_sum + hist_st
    
    hist_ae= torch.histc( map_ae, 10000, 0.0, 10.0)
    hist_ae_sum= hist_ae_sum + hist_ae



# index for 0.9 or 0.995 count the counted elements is quantile
quan_st_09= allcount*0.9
quan_st_0995= allcount*0.995

index_st_09= 0
index_st_0995= 0

count= 0
while count < quan_st_09 and index_st_09 < 10000:
    count= count+hist_st_sum[index_st_09]
    index_st_09= index_st_09+1

count= 0
while count < quan_st_0995 and index_st_0995 < 10000:
    count= count+hist_st_sum[index_st_0995]
    index_st_0995= index_st_0995+1

# ae
quan_ae_09= allcount*0.9
quan_ae_0995= allcount*0.995

index_ae_09= 0
index_ae_0995= 0

count= 0
while count < quan_ae_09 and index_ae_09 < 10000:
    count= count+hist_ae_sum[index_ae_09]
    index_ae_09= index_ae_09+1

count= 0
while count < quan_ae_0995 and index_ae_0995 < 10000:
    count= count+hist_ae_sum[index_ae_0995]
    index_ae_0995= index_ae_0995+1

'''
q_st_start = torch.quantile(hist_st_sum, q=0.9)
q_st_end = torch.quantile(hist_st_sum, q=0.995)
q_ae_start = torch.quantile(hist_ae_sum, q=0.9)
q_ae_end = torch.quantile(hist_ae_sum, q=0.995)
'''

index_st_09= index_st_09 / 1000.0
index_st_0995= index_st_0995 / 1000.0
index_ae_09= index_ae_09 / 1000.0
index_ae_0995= index_ae_0995 / 1000.0

print(index_st_09)
print(index_st_0995)
print(index_ae_09)
print(index_ae_0995)
input("Map...")

return index_st_09, index_st_0995, index_ae_09, index_ae_0995`