diff --git a/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_static/fx/run_clm.py b/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_static/fx/run_clm.py index 9ce477e9cab..001f256dc54 100644 --- a/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_static/fx/run_clm.py +++ b/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_static/fx/run_clm.py @@ -536,6 +536,22 @@ def compute_metrics(eval_preds): else None, ) + eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) + batch_size = eval_dataloader.batch_size + # Tune def eval_func_for_nc(model_tuned): trainer.model = model_tuned @@ -564,7 +580,7 @@ def eval_func_for_nc(model_tuned): conf = PostTrainingQuantConfig(accuracy_criterion=accuracy_criterion) q_model = quantization.fit(model, conf, - calib_dataloader=trainer.get_eval_dataloader(), + calib_dataloader=eval_dataloader, eval_func=eval_func_for_nc) q_model.save(training_args.output_dir) exit(0) @@ -582,7 +598,7 @@ def eval_func_for_nc(model_tuned): from neural_compressor.config import BenchmarkConfig from neural_compressor import benchmark b_conf = BenchmarkConfig(warmup=5, iteration=100, cores_per_instance=4, num_of_instance=1) - benchmark.fit(new_model, b_conf, b_dataloader=trainer.get_eval_dataloader()) + benchmark.fit(new_model, b_conf, b_dataloader=eval_dataloader) else: eval_func_for_nc(new_model) diff --git a/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_weight_only/run_clm.py b/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_weight_only/run_clm.py index 052dc4bc5cc..1c174415fe2 100644 --- a/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_weight_only/run_clm.py +++ b/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_weight_only/run_clm.py @@ -552,6 +552,22 @@ def compute_metrics(eval_preds): else None, ) + eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) + batch_size = eval_dataloader.batch_size + # Tune def eval_func_for_nc(model_tuned): trainer.model = model_tuned @@ -592,7 +608,7 @@ def eval_func_for_nc(model_tuned): op_type_dict=op_type_dict) q_model = quantization.fit(model, conf, - calib_dataloader=trainer.get_eval_dataloader(), + calib_dataloader=eval_dataloader, eval_func=eval_func_for_nc) q_model.save(training_args.output_dir) exit(0) @@ -610,7 +626,7 @@ def eval_func_for_nc(model_tuned): from neural_compressor.config import BenchmarkConfig from neural_compressor import benchmark b_conf = BenchmarkConfig(warmup=5, iteration=100, cores_per_instance=4, num_of_instance=1) - benchmark.fit(new_model, b_conf, b_dataloader=trainer.get_eval_dataloader()) + benchmark.fit(new_model, b_conf, b_dataloader=eval_dataloader) else: eval_func_for_nc(new_model) diff --git a/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/fx/run_qa.py b/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/fx/run_qa.py index e8772805168..37656ba22cd 100644 --- a/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/fx/run_qa.py +++ b/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/fx/run_qa.py @@ -605,6 +605,19 @@ def compute_metrics(p: EvalPrediction): ) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size metric_name = "eval_f1" diff --git a/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/ipex/run_qa.py b/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/ipex/run_qa.py index fa364018f6f..1254acdcdbf 100644 --- a/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/ipex/run_qa.py +++ b/examples/pytorch/nlp/huggingface_models/question-answering/quantization/ptq_static/ipex/run_qa.py @@ -612,6 +612,19 @@ def compute_metrics(p: EvalPrediction): ) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size metric_name = "eval_f1" diff --git a/examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/fx/run_summarization.py b/examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/fx/run_summarization.py index 95a32ffa745..ce4bdb4e375 100644 --- a/examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/fx/run_summarization.py +++ b/examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/fx/run_summarization.py @@ -588,6 +588,22 @@ def compute_metrics(eval_preds): ) num_beams = data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams + eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) + batch_size = eval_dataloader.batch_size + def eval_func_for_nc(model): trainer.model = model results = trainer.evaluate( @@ -616,7 +632,7 @@ def eval_func_for_nc(model): conf = PostTrainingQuantConfig(approach="dynamic") q_model = quantization.fit(model, conf, - calib_dataloader=trainer.get_eval_dataloader(), + calib_dataloader=eval_dataloader, eval_func=eval_func_for_nc) q_model.save(training_args.output_dir) exit(0) @@ -634,7 +650,7 @@ def eval_func_for_nc(model): from neural_compressor.config import BenchmarkConfig from neural_compressor import benchmark b_conf = BenchmarkConfig(warmup=5, iteration=100, cores_per_instance=4, num_of_instance=1) - benchmark.fit(new_model, b_conf, b_dataloader=trainer.get_eval_dataloader()) + benchmark.fit(new_model, b_conf, b_dataloader=eval_dataloader) else: eval_func_for_nc(new_model) diff --git a/examples/pytorch/nlp/huggingface_models/text-classification/export/fx/run_glue.py b/examples/pytorch/nlp/huggingface_models/text-classification/export/fx/run_glue.py index 7e8a0805af4..d93c0d2e06c 100644 --- a/examples/pytorch/nlp/huggingface_models/text-classification/export/fx/run_glue.py +++ b/examples/pytorch/nlp/huggingface_models/text-classification/export/fx/run_glue.py @@ -494,6 +494,19 @@ def compute_metrics(p: EvalPrediction): ) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size def take_eval_steps(model, trainer, save_metrics=False): diff --git a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_dynamic/fx/run_glue.py b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_dynamic/fx/run_glue.py index 638592d28c0..8fced9c3f91 100644 --- a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_dynamic/fx/run_glue.py +++ b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_dynamic/fx/run_glue.py @@ -479,6 +479,19 @@ def compute_metrics(p: EvalPrediction): ) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size def take_eval_steps(model, trainer, save_metrics=False): diff --git a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_static/fx/run_glue.py b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_static/fx/run_glue.py index 303c9330a44..a344b2d0562 100644 --- a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_static/fx/run_glue.py +++ b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/ptq_static/fx/run_glue.py @@ -479,6 +479,19 @@ def compute_metrics(p: EvalPrediction): ) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size def take_eval_steps(model, trainer, save_metrics=False): diff --git a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/qat/fx/run_glue.py b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/qat/fx/run_glue.py index 8051e305a75..749ee226837 100644 --- a/examples/pytorch/nlp/huggingface_models/text-classification/quantization/qat/fx/run_glue.py +++ b/examples/pytorch/nlp/huggingface_models/text-classification/quantization/qat/fx/run_glue.py @@ -506,6 +506,19 @@ def compute_metrics(p: EvalPrediction): early_stopping_threshold)) eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) batch_size = eval_dataloader.batch_size def eval_func(model): diff --git a/examples/pytorch/nlp/huggingface_models/translation/quantization/ptq_dynamic/fx/run_translation.py b/examples/pytorch/nlp/huggingface_models/translation/quantization/ptq_dynamic/fx/run_translation.py index 652b4b3fec1..e8bc6ddab4b 100644 --- a/examples/pytorch/nlp/huggingface_models/translation/quantization/ptq_dynamic/fx/run_translation.py +++ b/examples/pytorch/nlp/huggingface_models/translation/quantization/ptq_dynamic/fx/run_translation.py @@ -543,6 +543,22 @@ def compute_metrics(eval_preds): compute_metrics=compute_metrics if training_args.predict_with_generate else None ) + eval_dataloader = trainer.get_eval_dataloader() + # transformer issue #1 + # for transformers 4.31.0: accelerate dataloader + # *** ValueError: batch_size attribute should not be set + # after DataLoaderShard is initialized + if eval_dataloader.batch_size is None: + def _build_inc_dataloader(dataloader): + class INCDataLoader: + __iter__ = dataloader.__iter__ + def __init__(self) -> None: + self.dataloader = dataloader + self.batch_size = dataloader.total_batch_size + return INCDataLoader() + eval_dataloader = _build_inc_dataloader(eval_dataloader) + batch_size = eval_dataloader.batch_size + results = {} max_length = ( training_args.generation_max_length @@ -576,7 +592,7 @@ def eval_func_for_nc(model): conf = PostTrainingQuantConfig(approach="dynamic") q_model = quantization.fit(model, conf, - calib_dataloader=trainer.get_eval_dataloader(), + calib_dataloader=eval_dataloader, eval_func=eval_func_for_nc) q_model.save(training_args.output_dir) exit(0) @@ -595,7 +611,7 @@ def eval_func_for_nc(model): from neural_compressor.config import BenchmarkConfig from neural_compressor import benchmark b_conf = BenchmarkConfig(warmup=5, iteration=100, cores_per_instance=4, num_of_instance=1) - benchmark.fit(new_model, b_conf, b_dataloader=trainer.get_eval_dataloader()) + benchmark.fit(new_model, b_conf, b_dataloader=eval_dataloader) else: eval_func_for_nc(new_model) exit(0)