Skip to content

Commit

Permalink
partial ut optimization (intel#772)
Browse files Browse the repository at this point in the history
  • Loading branch information
@chensuyue
chensuyue committed May 27, 2022
1 parent c8408f4 commit 431e8e2
Show file tree
Hide file tree
Showing 7 changed files with 95 additions and 112 deletions.
98 changes: 29 additions & 69 deletions test/engine/test_adaptor_engine.py
View file
Original file line number Diff line number Diff line change
@@ -1,59 +1,38 @@
import sys
import os
import unittest
import numpy as np
from neural_compressor.experimental import Quantization, Benchmark, common
from neural_compressor.model.engine_model import EngineModel
from neural_compressor.data import DATASETS
from neural_compressor.experimental import Quantization

def build_yaml():
fake_yaml = """
model:
name: bert
framework: engine
quantization:
calibration:
sampling_size: 10
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
input_shape: [[128], [128], [128]]
label_shape: [128,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
op_wise: {
'bert/encoder/layer_0/attention/self/query/BiasAdd':{
'activation': {'dtype': ['fp32']},
'weight': {'dtype': ['fp32'], 'granularity': ['per_tensor']}
}
}
evaluation:
accuracy:
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
postprocess:
transform:
LabelShift: -1
metric:
MSE:
compare_label: False
GLUE:
task: sst-2
performance:
warmup: 5
iteration: 10
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
configs:
num_of_instance: 1
cores_per_instance: 28
tuning:
exit_policy:
max_trials: 1
Expand All @@ -62,54 +41,35 @@ def build_yaml():
f.write(fake_yaml)

fake_yaml_bf16 = """
model:
model:
name: bert
framework: engine
quantization:
quantization:
dtype: bf16
calibration:
sampling_size: 10
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
input_shape: [[128], [128], [128]]
label_shape: [128,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
op_wise: {
'bert/encoder/layer_0/attention/self/query/BiasAdd':{
'activation': {'dtype': ['fp32']},
'weight': {'dtype': ['fp32'], 'granularity': ['per_tensor']}
}
}
evaluation:
evaluation:
accuracy:
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
postprocess:
transform:
LabelShift: -1
metric:
MSE:
compare_label: False
GLUE:
task: sst-2
performance:
warmup: 5
iteration: 10
dataloader:
dataset:
dummy_v2:
input_shape: [[324], [324], [324]]
label_shape: [324,2]
low: [1, 0, 0, 0]
high: [128, 1, 1, 128]
dtype: [int32, int32, int32, float32]
tuning:
configs:
num_of_instance: 1
cores_per_instance: 28
tuning:
exit_policy:
max_trials: 1
"""
Expand All @@ -123,13 +83,13 @@ def setUp(self):

def test_adaptor(self):
quantizer = Quantization('test.yaml')
quantizer.model = "/home/tensorflow/inc_ut/engine/bert_mlperf_2none.pb"
quantizer.model = "/home/tensorflow/inc_ut/engine/bert_mini_sst2.onnx"
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

def test_adaptor_bf16(self):
quantizer = Quantization('test_bf16.yaml')
quantizer.model = "/home/tensorflow/inc_ut/engine/bert_mlperf_2none.pb"
quantizer.model = "/home/tensorflow/inc_ut/engine/bert_mini_sst2.onnx"
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

Expand All @@ -138,4 +98,4 @@ def tearDown(self):
os.remove('test_bf16.yaml')

if __name__ == "__main__":
unittest.main()
unittest.main()
19 changes: 13 additions & 6 deletions test/test_adaptor_onnxrt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import torch
import torchvision
import yaml
import onnx
import numpy as np
from collections import OrderedDict
Expand Down Expand Up @@ -450,32 +449,35 @@ def test_adaptor(self):
conf.quantization.approach = 'post_training_dynamic_quant'
conf.quantization.calibration.sampling_size = 1
conf.evaluation.accuracy.metric = {'Accuracy': {}}
quantizer = Quantization()
quantizer = Quantization(conf)
quantizer.calib_dataloader = self.rename_dataloader
quantizer.eval_dataloader = self.rename_dataloader
quantizer.model = self.rename_model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

for fake_yaml in ["static.yaml", "dynamic.yaml"]:
quantizer = Quantization(fake_yaml)
quantizer.calib_dataloader = self.cv_dataloader
quantizer.eval_dataloader = self.cv_dataloader
quantizer.model = self.rn50_model
q_model = quantizer.fit()
eval_func(q_model)
self.assertNotEqual(q_model, None)

import copy
tmp_model = copy.deepcopy(self.rn50_model)
tmp_model.opset_import[0].version = 10
quantizer.model = tmp_model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)
tmp_model.opset_import.extend([onnx.helper.make_opsetid("", 11)])
quantizer.model = tmp_model
q_model = quantizer.fit()
self.assertEqual(q_model, None)
model = onnx.load('rn50_9.onnx')
quantizer.model = model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

framework_specific_info = {"device": "cpu",
"approach": "post_training_static_quant",
Expand All @@ -502,17 +504,21 @@ def test_adaptor(self):
quantizer = Quantization(fake_yaml)
quantizer.model = self.gather_model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

quantizer.model = self.matmul_model
q_model = quantizer.fit()
q_model = quantizer.fit() # error input shape test
self.assertEqual(q_model, None)

quantizer.eval_dataloader = self.matmul_dataloader
q_model = quantizer.fit()
q_model = quantizer.fit() # error input shape test
self.assertEqual(q_model, None)

quantizer.calib_dataloader = self.matmul_dataloader
quantizer.eval_dataloader = self.matmul_dataloader
quantizer.model = self.matmul_model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

options.onnxrt.graph_optimization.level = 'ENABLE_BASIC'
for fake_yaml in ["non_MSE.yaml"]:
Expand All @@ -521,14 +527,15 @@ def test_adaptor(self):
quantizer.eval_dataloader = self.cv_dataloader
quantizer.model = self.mb_v2_model
q_model = quantizer.fit()
eval_func(q_model)
self.assertNotEqual(q_model, None)

for fake_yaml in ["static.yaml"]:
quantizer = Quantization(fake_yaml)
quantizer.calib_dataloader = self.ir3_dataloader
quantizer.eval_dataloader = self.ir3_dataloader
quantizer.model = self.ir3_model
q_model = quantizer.fit()
self.assertNotEqual(q_model, None)

from neural_compressor.utils.utility import recover
model = recover(self.ir3_model, './nc_workspace/recover/history.snapshot', 0)
Expand Down
2 changes: 1 addition & 1 deletion test/test_basic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def build_fake_yaml2():
strategy:
name: basic
exit_policy:
max_trials: 10
max_trials: 3
accuracy_criterion:
relative: -0.01
workspace:
Expand Down
9 changes: 7 additions & 2 deletions test/test_bayesian.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ def build_fake_yaml():
inputs: x
outputs: op_to_store
device: cpu
quantization:
calibration:
sampling_size: 10
evaluation:
accuracy:
metric:
Expand Down Expand Up @@ -62,9 +65,9 @@ def build_fake_yaml2():
strategy:
name: bayesian
exit_policy:
timeout: 60
max_trials: 3
accuracy_criterion:
relative: -0.01
relative: 0.01
workspace:
path: saved
'''
Expand Down Expand Up @@ -272,6 +275,7 @@ def test_run_bayesian_one_trial(self):
quantizer.calib_dataloader = common.DataLoader(dataset)
quantizer.model = self.constant_graph
output_graph = quantizer.fit()
self.assertNotEqual(output_graph, None)

def test_run_bayesian_max_trials(self):

Expand All @@ -282,6 +286,7 @@ def test_run_bayesian_max_trials(self):
quantizer.calib_dataloader = common.DataLoader(dataset)
quantizer.model = self.test_graph
output_graph = quantizer.fit()
self.assertNotEqual(output_graph, None)

def test_bayesian_opt_class(self):
from neural_compressor.strategy.bayesian import BayesianOptimization
Expand Down
11 changes: 0 additions & 11 deletions test/test_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,17 +282,6 @@ def test_tuning(self):
helper(test)
self.assertRaises(RuntimeError, conf.Conf, 'fake_conf.yaml')

test = '''
model:
name: tuning_yaml
framework: mxnet
tuning:
accuracy_criterion:
relative: 0.01
'''
helper(test)
self.assertRaises(RuntimeError, conf.Conf, 'fake_conf.yaml')

test = '''
model:
name: tuning_yaml
Expand Down
Loading

0 comments on commit 431e8e2

Please sign in to comment.