Skip to content

Commit

Permalink
Remove HfConfig::datasetreferences in examples and tests
Browse files Browse the repository at this point in the history 
Remove references to HFConfig::dataset from input model in all examples
and tests. Moved the references to the dataset into data_configs and
updated references accordingly. This change also eliminates the need for
the constant __input_model_data_config__.
  • Loading branch information
@shaahji
shaahji committed Apr 26, 2024
1 parent 91c4c20 commit 46a6058
Show file tree
Hide file tree
Showing 34 changed files with 1,327 additions and 573 deletions.
63 changes: 46 additions & 17 deletions .azure_pipelines/performance_check/configs/bert.json
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,66 @@
{
"input_model":{
"input_model": {
"type": "PyTorchModel",
"config": {
"hf_config": {
"model_name": "Intel/bert-base-uncased-mrpc",
"task": "text-classification",
"dataset": {
"data_name":"glue",
"subset": "mrpc",
"split": "validation",
"input_cols": ["sentence1", "sentence2"],
"label_cols": ["label"],
"batch_size": 1,
"max_samples": 100
}
"task": "text-classification"
}
}
},
"data_configs": [
{
"name": "glue_mrpc",
"type": "HuggingfaceContainer",
"params_config": {
"batch_size": 1,
"max_samples": 100,
"data_name": "glue",
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"split": "validation",
"subset": "mrpc"
}
}
],
"evaluators": {
"common_evaluator": {
"metrics":[
"metrics": [
{
"name": "accuracy",
"type": "accuracy",
"backend": "huggingface_metrics",
"data_config": "glue_mrpc",
"sub_types": [
{"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.05}} ]
{
"name": "accuracy",
"priority": 1,
"goal": {
"type": "max-degradation",
"value": 0.05
}
}
]
},
{
"name": "latency",
"type": "latency",
"data_config": "glue_mrpc",
"sub_types": [
{"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 5}} ]
{
"name": "avg",
"priority": 2,
"goal": {
"type": "percent-min-improvement",
"value": 5
}
}
]
}
]
}
Expand All @@ -57,13 +86,13 @@
"type": "OnnxQuantization",
"config": {
"quant_preprocess": true,
"data_config": "__input_model_data_config__"
"data_config": "glue_mrpc"
}
},
"perf_tuning": {
"type": "OrtPerfTuning",
"config": {
"data_config": "__input_model_data_config__"
"data_config": "glue_mrpc"
}
}
},
Expand All @@ -79,6 +108,6 @@
"clean_cache": true,
"evaluator": "common_evaluator",
"cache_dir": "run_cache/olive/cache",
"output_dir" : "run_cache/olive/bert_ptq"
"output_dir": "run_cache/olive/bert_ptq"
}
}
70 changes: 52 additions & 18 deletions .azure_pipelines/performance_check/configs/bert_gpu.json
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,10 @@
{
"input_model":{
"input_model": {
"type": "PyTorchModel",
"config": {
"hf_config": {
"model_name": "Intel/bert-base-uncased-mrpc",
"task": "text-classification",
"dataset": {
"data_name":"glue",
"subset": "mrpc",
"split": "validation",
"input_cols": ["sentence1", "sentence2"],
"label_cols": ["label"],
"batch_size": 1,
"max_samples": 100
}
"task": "text-classification"
}
}
},
Expand All @@ -32,22 +23,58 @@
}
}
},
"data_configs": [
{
"name": "glue_mrpc",
"type": "HuggingfaceContainer",
"params_config": {
"batch_size": 1,
"max_samples": 100,
"data_name": "glue",
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"split": "validation",
"subset": "mrpc"
}
}
],
"evaluators": {
"common_evaluator": {
"metrics":[
"metrics": [
{
"name": "accuracy",
"type": "accuracy",
"backend": "huggingface_metrics",
"data_config": "glue_mrpc",
"sub_types": [
{"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}}
{
"name": "accuracy",
"priority": 1,
"goal": {
"type": "max-degradation",
"value": 0.01
}
}
]
},
{
"name": "latency",
"type": "latency",
"data_config": "glue_mrpc",
"sub_types": [
{"name": "avg", "priority": 2, "goal": {"type": "percent-min-improvement", "value": 20}}
{
"name": "avg",
"priority": 2,
"goal": {
"type": "percent-min-improvement",
"value": 20
}
}
]
}
]
Expand All @@ -71,13 +98,20 @@
"type": "OrtPerfTuning",
"config": {
"enable_cuda_graph": true,
"data_config": "__input_model_data_config__"
"data_config": "glue_mrpc"
}
}
},
"pass_flows": [
["conversion", "transformers_optimization", "perf_tuning"],
["conversion", "perf_tuning"]
[
"conversion",
"transformers_optimization",
"perf_tuning"
],
[
"conversion",
"perf_tuning"
]
],
"engine": {
"search_strategy": {
Expand All @@ -93,6 +127,6 @@
"target": "local_system",
"clean_cache": true,
"cache_dir": "run_cache/olive/cache",
"output_dir" : "run_cache/olive/bert_gpu"
"output_dir": "run_cache/olive/bert_gpu"
}
}
66 changes: 44 additions & 22 deletions .azure_pipelines/performance_check/configs/deberta.json
View file
Original file line number Diff line number Diff line change
@@ -1,44 +1,66 @@
{
"input_model":{
"input_model": {
"type": "PyTorchModel",
"config": {
"hf_config": {
"model_name": "microsoft/deberta-base-mnli",
"task": "text-classification",
"dataset": {
"data_name":"glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": ["premise", "hypothesis"],
"label_cols": ["label"],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
"task": "text-classification"
}
}
},
"data_configs": [
{
"name": "glue_mnli_matched",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
}
}
}
},
],
"evaluators": {
"common_evaluator": {
"metrics":[
"metrics": [
{
"name": "accuracy",
"type": "accuracy",
"backend": "huggingface_metrics",
"data_config": "glue_mnli_matched",
"sub_types": [
{"name": "accuracy", "priority": 1, "goal": {"type": "max-degradation", "value": 0.05}}

{
"name": "accuracy",
"priority": 1,
"goal": {
"type": "max-degradation",
"value": 0.05
}
}
]
},
{
"name": "latency",
"type": "latency",
"data_config": "glue_mnli_matched",
"sub_types": [
{"name": "avg", "priority": 2}
{
"name": "avg",
"priority": 2
}
]
}
]
Expand All @@ -60,13 +82,13 @@
"config": {
"quant_mode": "dynamic",
"quant_preprocess": true,
"data_config": "__input_model_data_config__"
"data_config": "glue_mnli_matched"
}
},
"perf_tuning": {
"type": "OrtPerfTuning",
"config": {
"data_config": "__input_model_data_config__"
"data_config": "glue_mnli_matched"
}
}
},
Expand All @@ -82,6 +104,6 @@
"clean_cache": true,
"evaluator": "common_evaluator",
"cache_dir": "run_cache/olive/cache",
"output_dir" : "run_cache/olive/microsoft-deberta"
"output_dir": "run_cache/olive/microsoft-deberta"
}
}
Loading

0 comments on commit 46a6058

Please sign in to comment.