From af104953ff062421b9830e29a5c113b1d4b43eb6 Mon Sep 17 00:00:00 2001 From: Zixuan Cheng <110808245+violetch24@users.noreply.github.com> Date: Mon, 29 May 2023 21:28:51 +0800 Subject: [PATCH] add PT 3dunet model (#811) Signed-off-by: Cheng, Zixuan --- examples/.config/model_params_pytorch.json | 7 + examples/README.md | 6 + .../3d-unet/quantization/ptq/fx/.dockerignore | 1 + .../3d-unet/quantization/ptq/fx/.gitignore | 1 + .../3d-unet/quantization/ptq/fx/Makefile | 204 ++++++++++++++++ .../3d-unet/quantization/ptq/fx/README.md | 76 ++++++ .../quantization/ptq/fx/Task043_BraTS_2019.py | 135 +++++++++++ .../quantization/ptq/fx/accuracy-brats.py | 158 +++++++++++++ .../3d-unet/quantization/ptq/fx/brats_QSL.py | 58 +++++ .../ptq/fx/brats_cal_images_list.txt | 40 ++++ .../ptq/fx/folds/fold0_validation.txt | 67 ++++++ .../ptq/fx/folds/fold1_validation.txt | 67 ++++++ .../ptq/fx/folds/fold2_validation.txt | 67 ++++++ .../ptq/fx/folds/fold3_validation.txt | 67 ++++++ .../ptq/fx/folds/fold4_validation.txt | 67 ++++++ .../3d-unet/quantization/ptq/fx/mlperf.conf | 65 +++++ .../quantization/ptq/fx/onnxruntime_SUT.py | 61 +++++ .../3d-unet/quantization/ptq/fx/ov_SUT.py | 85 +++++++ .../3d-unet/quantization/ptq/fx/preprocess.py | 123 ++++++++++ .../quantization/ptq/fx/pytorch_SUT.py | 77 ++++++ .../quantization/ptq/fx/requirements.txt | 8 + .../3d-unet/quantization/ptq/fx/run.py | 222 ++++++++++++++++++ .../quantization/ptq/fx/run_benchmark.sh | 69 ++++++ .../3d-unet/quantization/ptq/fx/run_tuning.sh | 49 ++++ .../3d-unet/quantization/ptq/fx/tf_SUT.py | 73 ++++++ .../quantization/ptq/fx/unet_onnx_to_tf.py | 59 +++++ .../ptq/fx/unet_pytorch_to_onnx.py | 76 ++++++ .../3d-unet/quantization/ptq/fx/user.conf | 6 + 28 files changed, 1994 insertions(+) create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.dockerignore create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.gitignore create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Makefile create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/README.md create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Task043_BraTS_2019.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/accuracy-brats.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_QSL.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_cal_images_list.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold0_validation.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold1_validation.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold2_validation.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold3_validation.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold4_validation.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/mlperf.conf create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/onnxruntime_SUT.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/ov_SUT.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/preprocess.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/pytorch_SUT.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/requirements.txt create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_benchmark.sh create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_tuning.sh create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/tf_SUT.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_onnx_to_tf.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_pytorch_to_onnx.py create mode 100644 examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/user.conf diff --git a/examples/.config/model_params_pytorch.json b/examples/.config/model_params_pytorch.json index d665390e1ac..875e3a35b05 100644 --- a/examples/.config/model_params_pytorch.json +++ b/examples/.config/model_params_pytorch.json @@ -311,6 +311,13 @@ "batch_size": 64, "main_script": "run_glue.py" }, + "3dunet": { + "model_src_dir": "image_recognition/3d-unet/quantization/ptq/fx", + "dataset_location": "/tf_dataset/dataset/mlperf_3dunet/build", + "input_model": "/tf_dataset/pytorch/mlperf_3dunet/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1", + "batch_size": 100, + "main_script": "run.py" + }, "rnnt": { "model_src_dir": "speech_recognition/rnnt/quantization/ptq_dynamic/eager", "dataset_location": "/tf_dataset/pytorch/rnnt/convert_dataset/", diff --git a/examples/README.md b/examples/README.md index 80cd7601b21..41bacc3b461 100644 --- a/examples/README.md +++ b/examples/README.md @@ -419,6 +419,12 @@ Intel® Neural Compressor validated examples with multiple compression technique Post-Training Static Quantization fx + + 3D-UNet + Image Recognition + Post-Training Static Quantization + fx + SSD ResNet34 Object Detection diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.dockerignore b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.dockerignore new file mode 100644 index 00000000000..567609b1234 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.dockerignore @@ -0,0 +1 @@ +build/ diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.gitignore b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.gitignore new file mode 100644 index 00000000000..567609b1234 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Makefile b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Makefile new file mode 100644 index 00000000000..50850421418 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Makefile @@ -0,0 +1,204 @@ +# Copyright (c) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SHELL := /bin/bash + +MAKEFILE_NAME := $(lastword $(MAKEFILE_LIST)) +UNAME := $(shell whoami) +UID := $(shell id -u `whoami`) +GROUPNAME := $(shell id -gn `whoami`) +GROUPID := $(shell id -g `whoami`) + +HOST_VOL ?= ${PWD} +CONTAINER_VOL ?= /workspace + +BUILD_DIR := build +ifndef DOWNLOAD_DATA_DIR + export DOWNLOAD_DATA_DIR := $(HOST_VOL)/$(BUILD_DIR)/MICCAI_BraTS_2019_Data_Training +endif +RAW_DATA_DIR := $(BUILD_DIR)/raw_data +PREPROCESSED_DATA_DIR := $(BUILD_DIR)/preprocessed_data +POSTPROCESSED_DATA_DIR := $(BUILD_DIR)/postprocessed_data +MODEL_DIR := $(BUILD_DIR)/model +RESULT_DIR := $(BUILD_DIR)/result +MLPERF_CONF := $(BUILD_DIR)/mlperf.conf +PYTORCH_MODEL := $(RESULT_DIR)/fold_1.zip +ONNX_MODEL := $(MODEL_DIR)/224_224_160.onnx +ONNX_DYNAMIC_BS_MODEL := $(MODEL_DIR)/224_224_160_dynamic_bs.onnx +TF_MODEL := $(MODEL_DIR)/224_224_160.pb +OPENVINO_MODEL := $(MODEL_DIR)/brats_model_checkpoint_final_fold1_H224_W224_D160_C4.bin +OPENVINO_MODEL_METADATA := $(MODEL_DIR)/brats_model_checkpoint_final_fold1_H224_W224_D160_C4.xml + +# Env variables needed by nnUnet +export nnUNet_raw_data_base=$(RAW_DATA_DIR) +export nnUNet_preprocessed=$(PREPROCESSED_DATA_DIR) +export RESULTS_FOLDER=$(RESULT_DIR) + +.PHONY: setup +setup: check_download_data_dir create_directories + @echo "Running basic setup..." + @if [ ! -e $(MLPERF_CONF) ]; then \ + cp ../../../mlperf.conf $(MLPERF_CONF); \ + fi + @$(MAKE) -f $(MAKEFILE_NAME) init_submodule + @$(MAKE) -f $(MAKEFILE_NAME) download_model + +.PHONY: check_download_data_dir +check_download_data_dir: + @if [ ! -e $(DOWNLOAD_DATA_DIR) ]; then \ + echo "Please set environment variable DOWNLOAD_DATA_DIR to " && false ; \ + fi + +.PHONY: create_directories +create_directories: + @if [ ! -e $(BUILD_DIR) ]; then \ + mkdir $(BUILD_DIR); \ + fi + @if [ ! -e $(MODEL_DIR) ]; then \ + mkdir $(MODEL_DIR); \ + fi + @if [ ! -e $(RESULT_DIR) ]; then \ + mkdir $(RESULT_DIR); \ + fi + +.PHONY: init_submodule +init_submodule: + @echo "Initialize nnUnet submodule.." + #@git submodule update --init nnUnet + +.PHONY: download_model +download_model: + @echo "Download models..." + @$(MAKE) -f $(MAKEFILE_NAME) download_pytorch_model + @$(MAKE) -f $(MAKEFILE_NAME) download_onnx_model + @$(MAKE) -f $(MAKEFILE_NAME) download_tf_model + @$(MAKE) -f $(MAKEFILE_NAME) download_openvino_model + +.PHONY: download_pytorch_model +download_pytorch_model: create_directories + @echo "Downloading PyTorch model from Zenodo..." + @if [ ! -e $(PYTORCH_MODEL) ]; then \ + wget -O $(PYTORCH_MODEL) https://zenodo.org/record/3904106/files/fold_1.zip?download=1 \ + && cd $(RESULT_DIR) && unzip -o fold_1.zip; \ + fi + +.PHONY: download_onnx_model +download_onnx_model: create_directories + @echo "Downloading ONNX model from Zenodo..." + @if [ ! -e $(ONNX_MODEL) ]; then \ + wget -O $(ONNX_MODEL) https://zenodo.org/record/3928973/files/224_224_160.onnx?download=1; \ + fi + @if [ ! -e $(ONNX_DYNAMIC_BS_MODEL) ]; then \ + wget -O $(ONNX_DYNAMIC_BS_MODEL) https://zenodo.org/record/3928973/files/224_224_160_dyanmic_bs.onnx?download=1; \ + fi + +.PHONY: download_tf_model +download_tf_model: create_directories + @echo "Downloading TF model from Zenodo..." + @if [ ! -e $(TF_MODEL) ]; then \ + wget -O $(TF_MODEL) https://zenodo.org/record/3928991/files/224_224_160.pb?download=1; \ + fi + +.PHONY: download_openvino_model +download_openvino_model: create_directories + @echo "Downloading OpenVINO model from Zenodo..." + @if [ ! -e $(OPENVINO_MODEL) ]; then \ + wget -O $(OPENVINO_MODEL) https://zenodo.org/record/3929002/files/brats_model_checkpoint_final_fold1_H224_W224_D160_C4.bin?download=1; \ + fi + @if [ ! -e $(OPENVINO_MODEL_METADATA) ]; then \ + wget -O $(OPENVINO_MODEL_METADATA) https://zenodo.org/record/3929002/files/brats_model_checkpoint_final_fold1_H224_W224_D160_C4.xml?download=1; \ + fi + +.PHONY: convert_onnx_model +convert_onnx_model: download_pytorch_model + @echo "Converting PyTorch model to ONNX model..." + @if [ ! -e $(ONNX_MODEL) ]; then \ + python3 unet_pytorch_to_onnx.py; \ + fi + +.PHONY: convert_tf_model +convert_tf_model: convert_onnx_model + @echo "Converting ONNX model to TF model..." + @if [ ! -e $(TF_MODEL) ]; then \ + python3 unet_onnx_to_tf.py; \ + fi + +.PHONY: preprocess_data +preprocess_data: create_directories + @echo "Restructuring raw data to $(RAW_DATA_DIR)..." + @if [ ! -e $(RAW_DATA_DIR) ]; then \ + mkdir $(RAW_DATA_DIR); \ + fi + @python3 Task043_BraTS_2019.py --downloaded_data_dir $(DOWNLOAD_DATA_DIR) + @echo "Preprocessing and saving preprocessed data to $(PREPROCESSED_DATA_DIR)..." + @if [ ! -e $(PREPROCESSED_DATA_DIR) ]; then \ + mkdir $(PREPROCESSED_DATA_DIR); \ + fi + @python3 preprocess.py + +.PHONY: mkdir_postprocessed_data +mkdir_postprocessed_data: + @if [ ! -e $(POSTPROCESSED_DATA_DIR) ]; then \ + mkdir $(POSTPROCESSED_DATA_DIR); \ + fi + +.PHONY: run_pytorch_performance +run_pytorch_performance: + @python3 run.py --backend=pytorch + +.PHONY: run_pytorch_accuracy +run_pytorch_accuracy: mkdir_postprocessed_data + @python3 run.py --backend=pytorch --accuracy + +.PHONY: run_pytorch_NC_tuning +run_pytorch_NC_tuning: mkdir_postprocessed_data + @python3 run.py --backend=pytorch --accuracy --tune --mlperf_conf=./mlperf.conf + +.PHONY: run_onnxruntime_performance +run_onnxruntime_performance: + @python3 run.py --backend=onnxruntime --model=build/model/224_224_160.onnx + +.PHONY: run_onnxruntime_accuracy +run_onnxruntime_accuracy: mkdir_postprocessed_data + @python3 run.py --backend=onnxruntime --model=build/model/224_224_160.onnx --accuracy + +.PHONY: run_tf_performance +run_tf_performance: + @python3 run.py --backend=tf --model=build/model/224_224_160.pb + +.PHONY: run_tf_accuracy +run_tf_accuracy: mkdir_postprocessed_data + @python3 run.py --backend=tf --model=build/model/224_224_160.pb --accuracy + +.PHONY: evaluate +evaluate: + @python3 accuracy-brats.py + +.PHONY: clean +clean: + @rm -rf build diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/README.md b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/README.md new file mode 100644 index 00000000000..a36f7e2440f --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/README.md @@ -0,0 +1,76 @@ +Step-by-Step +============ + +This example is used to demonstrate the steps of reproducing quantization and benchmarking results with Intel® Neural Compressor. + +The 3D-Unet source code comes from [mlperf](https://github.com/mlcommons/inference/tree/v1.0.1/vision/medical_imaging/3d-unet), commit SHA is **b7e8f0da170a421161410d18e5d2a05d75d6bccf**; [nnUnet](https://github.com/MIC-DKFZ/nnUNet) commit SHA is **b38c69b345b2f60cd0d053039669e8f988b0c0af**. Users could diff them with this example to know which changes have been made to integrate with Intel® Neural Compressor.. + +The model is performing on [BraTS 2019](https://www.med.upenn.edu/cbica/brats2019/data.html) brain tumor segmentation task. + +# Prerequisite +## 1. Environment +Python 3.6 or higher version is recommended. +The dependent packages are all in requirements, please install as following. +```shell +cd examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx +pip install -r requirements.txt +``` +## 2. Preprocess Dataset +```shell + # download BraTS 2019 from https://www.med.upenn.edu/cbica/brats2019/data.html + export DOWNLOAD_DATA_DIR= # point to location of downloaded BraTS 2019 Training dataset. + + # install dependency required by data preprocessing script + git clone https://github.com/MIC-DKFZ/nnUNet.git --recursive + cd nnUNet/ + git checkout b38c69b345b2f60cd0d053039669e8f988b0c0af + # replace sklearn in the older version with scikit-learn + sed -i 's/sklearn/scikit-learn/g' setup.py + python setup.py install + cd .. + + # download pytorch model + make download_pytorch_model + + # generate preprocessed data + make preprocess_data + + # create postprocess dir + make mkdir_postprocessed_data + + # generate calibration preprocessed data + python preprocess.py --preprocessed_data_dir=./build/calib_preprocess/ --validation_fold_file=./brats_cal_images_list.txt + + # install mlperf loadgen required by tuning script + git clone https://github.com/mlcommons/inference.git --recursive + cd inference + git checkout b7e8f0da170a421161410d18e5d2a05d75d6bccf + cd loadgen + pip install absl-py + python setup.py install + cd ../.. +``` + +# Run +## 1. Quantization + +```shell + make run_pytorch_NC_tuning +``` + + or + +```shell + python run.py --model_dir=build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1 --backend=pytorch --accuracy --preprocessed_data_dir=build/preprocessed_data/ --mlperf_conf=./mlperf.conf --tune +``` +## 2. Benchmark +```bash +# int8 +sh run_benchmark.sh --int8=true --input_model=build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1 --dataset_location=build/preprocessed_data/ +# fp32 +sh run_benchmark.sh --input_model=build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1 --dataset_location=build/preprocessed_data/ +``` +## 3. Model Baseline +| model | framework | accuracy | dataset | model link | model source | precision | +| - | - | - | - | - | - | - | +| 3D-Unet | PyTorch | **mean = 0.85300** (whole tumor = 0.9141, tumor core = 0.8679, enhancing tumor = 0.7770) | [Fold 1](folds/fold1_validation.txt) of [BraTS 2019](https://www.med.upenn.edu/cbica/brats2019/data.html) Training Dataset | [from zenodo](https://zenodo.org/record/3904106) | Trained in PyTorch using codes from[nnUnet](https://github.com/MIC-DKFZ/nnUNet) on [Fold 0](folds/fold0_validation.txt), [Fold 2](folds/fold2_validation.txt), [Fold 3](folds/fold3_validation.txt), and [Fold 4](folds/fold4_validation.txt) of [BraTS 2019](https://www.med.upenn.edu/cbica/brats2019/data.html) Training Dataset. | fp32 | diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Task043_BraTS_2019.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Task043_BraTS_2019.py new file mode 100644 index 00000000000..6c196077bd1 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/Task043_BraTS_2019.py @@ -0,0 +1,135 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is copied from nnUnet/nnunet/dataset_conversion/Task043_BraTS_2019.py, except that +# the validation/test set part is removed and downloaded_data_dir is now configurable. + +import argparse +import numpy as np +from collections import OrderedDict +import os +import sys + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) + +from batchgenerators.utilities.file_and_folder_operations import * +from nnunet.paths import nnUNet_raw_data +import SimpleITK as sitk +import shutil + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--downloaded_data_dir", default="build/MICCAI_BraTS_2019_Data_Training", help="path to MICCAI_BraTS_2019_Data_Training") + args = parser.parse_args() + return args + +def copy_BraTS_segmentation_and_convert_labels(in_file, out_file): + # use this for segmentation only!!! + # nnUNet wants the labels to be continuous. BraTS is 0, 1, 2, 4 -> we make that into 0, 1, 2, 3 + img = sitk.ReadImage(in_file) + img_npy = sitk.GetArrayFromImage(img) + + uniques = np.unique(img_npy) + for u in uniques: + if u not in [0, 1, 2, 4]: + raise RuntimeError('unexpected label') + + seg_new = np.zeros_like(img_npy) + seg_new[img_npy == 4] = 3 + seg_new[img_npy == 2] = 1 + seg_new[img_npy == 1] = 2 + img_corr = sitk.GetImageFromArray(seg_new) + img_corr.CopyInformation(img) + sitk.WriteImage(img_corr, out_file) + +def main(): + + args = get_args() + + """ + REMEMBER TO CONVERT LABELS BACK TO BRATS CONVENTION AFTER PREDICTION! + """ + + task_name = "Task043_BraTS2019" + downloaded_data_dir = args.downloaded_data_dir + + target_base = join(nnUNet_raw_data, task_name) + target_imagesTr = join(target_base, "imagesTr") + target_imagesVal = join(target_base, "imagesVal") + target_imagesTs = join(target_base, "imagesTs") + target_labelsTr = join(target_base, "labelsTr") + + maybe_mkdir_p(target_imagesTr) + maybe_mkdir_p(target_imagesVal) + maybe_mkdir_p(target_imagesTs) + maybe_mkdir_p(target_labelsTr) + + patient_names = [] + for tpe in ["HGG", "LGG"]: + cur = join(downloaded_data_dir, tpe) + for p in subdirs(cur, join=False): + patdir = join(cur, p) + patient_name = tpe + "__" + p + patient_names.append(patient_name) + t1 = join(patdir, p + "_t1.nii.gz") + t1c = join(patdir, p + "_t1ce.nii.gz") + t2 = join(patdir, p + "_t2.nii.gz") + flair = join(patdir, p + "_flair.nii.gz") + seg = join(patdir, p + "_seg.nii.gz") + + assert all([ + isfile(t1), + isfile(t1c), + isfile(t2), + isfile(flair), + isfile(seg) + ]), "%s" % patient_name + + shutil.copy(t1, join(target_imagesTr, patient_name + "_0000.nii.gz")) + shutil.copy(t1c, join(target_imagesTr, patient_name + "_0001.nii.gz")) + shutil.copy(t2, join(target_imagesTr, patient_name + "_0002.nii.gz")) + shutil.copy(flair, join(target_imagesTr, patient_name + "_0003.nii.gz")) + + copy_BraTS_segmentation_and_convert_labels(seg, join(target_labelsTr, patient_name + ".nii.gz")) + + json_dict = OrderedDict() + json_dict['name'] = "BraTS2019" + json_dict['description'] = "nothing" + json_dict['tensorImageSize'] = "4D" + json_dict['reference'] = "see BraTS2019" + json_dict['licence'] = "see BraTS2019 license" + json_dict['release'] = "0.0" + json_dict['modality'] = { + "0": "T1", + "1": "T1ce", + "2": "T2", + "3": "FLAIR" + } + json_dict['labels'] = { + "0": "background", + "1": "edema", + "2": "non-enhancing", + "3": "enhancing", + } + json_dict['numTraining'] = len(patient_names) + json_dict['numTest'] = 0 + json_dict['training'] = [{'image': "./imagesTr/%s.nii.gz" % i, "label": "./labelsTr/%s.nii.gz" % i} for i in + patient_names] + json_dict['test'] = [] + + save_json(json_dict, join(target_base, "dataset.json")) + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/accuracy-brats.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/accuracy-brats.py new file mode 100644 index 00000000000..874ce8517a6 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/accuracy-brats.py @@ -0,0 +1,158 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# Copyright (c) 2021 INTEL CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import numpy as np +import os +import pickle +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "nnUnet")) + +from multiprocessing import Pool +from nnunet.evaluation.region_based_evaluation import evaluate_regions, get_brats_regions +from nnunet.inference.segmentation_export import save_segmentation_nifti_from_softmax + +dtype_map = { + "int8": np.int8, + "int16": np.int16, + "int32": np.int32, + "int64": np.int64, + "float16": np.float16, + "float32": np.float32, + "float64": np.float64 +} + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--log_file", default="build/logs/mlperf_log_accuracy.json", help="Path to accuracy log json file") + parser.add_argument("--output_dtype", default="float16", choices=dtype_map.keys(), help="Output data type") + parser.add_argument("--preprocessed_data_dir", default="build/preprocessed_data", help="Path to the directory containing preprocessed data") + parser.add_argument("--postprocessed_data_dir", default="build/postprocessed_data", help="Path to the directory containing postprocessed data") + parser.add_argument("--label_data_dir", default="build/raw_data/nnUNet_raw_data/Task043_BraTS2019/labelsTr", + help="Path to the directory containing ground truth labels") + parser.add_argument("--num_threads_nifti_save", type=int, default=12, help="Number of threads to run the postprocessing with") + args = parser.parse_args() + return args + +def save_predictions_MLPerf(predictions, output_folder, output_files, dictionaries, num_threads_nifti_save, all_in_gpu, force_separate_z=None, interp_order=3, interp_order_z=0): + print("Saving predictions...") + pool = Pool(num_threads_nifti_save) + results = [] + for i, output_filename in enumerate(output_files): + print(i, "/", len(output_files)) + output_filename = os.path.join(output_folder, output_filename + ".nii.gz") + softmax_mean = predictions[i] + dct = dictionaries[i] + bytes_per_voxel = 4 + if all_in_gpu: + bytes_per_voxel = 2 # if all_in_gpu then the return value is half (float16) + if np.prod(softmax_mean.shape) > (2e9 / bytes_per_voxel * 0.85): # * 0.85 just to be save + print( + "This output is too large for python process-process communication. Saving output temporarily to disk") + np.save(output_filename[:-7] + ".npy", softmax_mean) + softmax_mean = output_filename[:-7] + ".npy" + + results.append(pool.starmap_async(save_segmentation_nifti_from_softmax, + ((softmax_mean, output_filename, dct, interp_order, None, None, None, + None, None, force_separate_z, interp_order_z),) + )) + _ = [i.get() for i in results] + + pool.close() + pool.join() + + del predictions + +def load_loadgen_log(log_file, result_dtype, dictionaries): + with open(log_file) as f: + predictions = json.load(f) + + assert len(predictions) == len(dictionaries), "Number of predictions does not match number of samples in validation set!" + + padded_shape = [224, 224, 160] + results = [None for i in range(len(predictions))] + for prediction in predictions: + qsl_idx = prediction["qsl_idx"] + assert qsl_idx >= 0 and qsl_idx < len(predictions), "Invalid qsl_idx!" + raw_shape = list(dictionaries[qsl_idx]["size_after_cropping"]) + # Remove the padded part + pad_before = [(p - r) // 2 for p, r in zip(padded_shape, raw_shape)] + pad_after = [-(p - r - b) for p, r, b in zip(padded_shape, raw_shape, pad_before)] + result_shape = (4,) + tuple(padded_shape) + result = np.frombuffer(bytes.fromhex(prediction["data"]), result_dtype).reshape(result_shape).astype(np.float16) + results[qsl_idx] = result[:, pad_before[0]:pad_after[0], pad_before[1]:pad_after[1], pad_before[2]:pad_after[2]] + + assert all([i is not None for i in results]), "Missing some results!" + + return results + +def main(): + args = get_args() + log_file = args.log_file + preprocessed_data_dir = args.preprocessed_data_dir + output_folder = args.postprocessed_data_dir + ground_truths = args.label_data_dir + output_dtype = dtype_map[args.output_dtype] + num_threads_nifti_save = args.num_threads_nifti_save + all_in_gpu = "None" + force_separate_z = None + interp_order = 3 + interp_order_z = 0 + + # Load necessary metadata. + print("Loading necessary metadata...") + with open(os.path.join(preprocessed_data_dir, "preprocessed_files.pkl"), "rb") as f: + preprocessed_files = pickle.load(f) + dictionaries = [] + for preprocessed_file in preprocessed_files: + with open(os.path.join(preprocessed_data_dir, preprocessed_file + ".pkl"), "rb") as f: + dct = pickle.load(f)[1] + dictionaries.append(dct) + + # Load predictions from loadgen accuracy log. + print("Loading loadgen accuracy log...") + predictions = load_loadgen_log(log_file, output_dtype, dictionaries) + + # Save predictions + # This runs in multiprocess + print("Running postprocessing with multiple threads...") + save_predictions_MLPerf(predictions, output_folder, preprocessed_files, dictionaries, num_threads_nifti_save, all_in_gpu, force_separate_z, interp_order, interp_order_z) + + # Run evaluation + print("Running evaluation...") + evaluate_regions(output_folder, ground_truths, get_brats_regions()) + + # Load evaluation summary + print("Loading evaluation summary...") + with open(os.path.join(output_folder, "summary.csv")) as f: + for line in f: + words = line.split(",") + if words[0] == "mean": + whole = float(words[1]) + core = float(words[2]) + enhancing = float(words[3]) + mean = (whole + core + enhancing) / 3 + print("Accuracy: mean = {:.5f}, whole tumor = {:.4f}, tumor core = {:.4f}, enhancing tumor = {:.4f}".format(mean, whole, core, enhancing)) + return mean + + print("Done!") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_QSL.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_QSL.py new file mode 100644 index 00000000000..3fc88c455c8 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_QSL.py @@ -0,0 +1,58 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle +import sys +sys.path.insert(0, os.getcwd()) + +import mlperf_loadgen as lg + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) +from nnUNet.nnunet.inference.predict import preprocess_multithreaded + +class BraTS_2019_QSL(): + def __init__(self, preprocessed_data_dir, perf_count): + print("Constructing QSL...") + self.preprocessed_data_dir = preprocessed_data_dir + with open(os.path.join(self.preprocessed_data_dir, "preprocessed_files.pkl"), "rb") as f: + self.preprocess_files = pickle.load(f) + + self.count = len(self.preprocess_files) + self.perf_count = perf_count if perf_count is not None else self.count + print("Found {:d} preprocessed files".format(self.count)) + print("Using performance count = {:d}".format(self.perf_count)) + + self.loaded_files = {} + self.qsl = lg.ConstructQSL(self.count, self.perf_count, self.load_query_samples, self.unload_query_samples) + print("Finished constructing QSL.") + + def load_query_samples(self, sample_list): + for sample_id in sample_list: + file_name = self.preprocess_files[sample_id] + print("Loading file {:}".format(file_name)) + with open(os.path.join(self.preprocessed_data_dir, "{:}.pkl".format(file_name)), "rb") as f: + self.loaded_files[sample_id] = pickle.load(f)[0] + + def unload_query_samples(self, sample_list): + for sample_id in sample_list: + del self.loaded_files[sample_id] + + def get_features(self, sample_id): + return self.loaded_files[sample_id] + +def get_brats_QSL(preprocessed_data_dir="build/preprocessed_data", perf_count=None): + return BraTS_2019_QSL(preprocessed_data_dir, perf_count) diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_cal_images_list.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_cal_images_list.txt new file mode 100644 index 00000000000..69276e67b6a --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/brats_cal_images_list.txt @@ -0,0 +1,40 @@ +HGG__BraTS19_2013_18_1 +HGG__BraTS19_2013_20_1 +HGG__BraTS19_CBICA_AAP_1 +HGG__BraTS19_CBICA_ABN_1 +HGG__BraTS19_CBICA_ABO_1 +HGG__BraTS19_CBICA_ALU_1 +HGG__BraTS19_CBICA_ANZ_1 +HGG__BraTS19_CBICA_APY_1 +HGG__BraTS19_CBICA_AQJ_1 +HGG__BraTS19_CBICA_AQZ_1 +HGG__BraTS19_CBICA_ASN_1 +HGG__BraTS19_CBICA_ASY_1 +HGG__BraTS19_CBICA_AUW_1 +HGG__BraTS19_CBICA_AXJ_1 +HGG__BraTS19_CBICA_AXM_1 +HGG__BraTS19_CBICA_AYG_1 +HGG__BraTS19_CBICA_AYU_1 +HGG__BraTS19_CBICA_AZD_1 +HGG__BraTS19_CBICA_BAX_1 +HGG__BraTS19_CBICA_BGR_1 +HGG__BraTS19_CBICA_BHV_1 +HGG__BraTS19_TCIA01_235_1 +HGG__BraTS19_TCIA02_394_1 +HGG__BraTS19_TCIA02_473_1 +HGG__BraTS19_TCIA02_606_1 +HGG__BraTS19_TCIA03_419_1 +HGG__BraTS19_TCIA04_192_1 +HGG__BraTS19_TCIA04_479_1 +HGG__BraTS19_TCIA06_372_1 +HGG__BraTS19_TCIA08_278_1 +LGG__BraTS19_2013_28_1 +LGG__BraTS19_TCIA09_462_1 +LGG__BraTS19_TCIA10_130_1 +LGG__BraTS19_TCIA10_202_1 +LGG__BraTS19_TCIA10_346_1 +LGG__BraTS19_TCIA10_387_1 +LGG__BraTS19_TCIA10_628_1 +LGG__BraTS19_TCIA12_470_1 +LGG__BraTS19_TCIA13_621_1 +LGG__BraTS19_TCIA13_653_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold0_validation.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold0_validation.txt new file mode 100644 index 00000000000..57eeeb651c5 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold0_validation.txt @@ -0,0 +1,67 @@ +HGG__BraTS19_2013_22_1 +HGG__BraTS19_2013_23_1 +HGG__BraTS19_2013_3_1 +HGG__BraTS19_2013_5_1 +HGG__BraTS19_2013_7_1 +HGG__BraTS19_CBICA_AAB_1 +HGG__BraTS19_CBICA_AAL_1 +HGG__BraTS19_CBICA_ABN_1 +HGG__BraTS19_CBICA_ALU_1 +HGG__BraTS19_CBICA_AME_1 +HGG__BraTS19_CBICA_ANG_1 +HGG__BraTS19_CBICA_AOC_1 +HGG__BraTS19_CBICA_AOD_1 +HGG__BraTS19_CBICA_APZ_1 +HGG__BraTS19_CBICA_AQD_1 +HGG__BraTS19_CBICA_AQJ_1 +HGG__BraTS19_CBICA_AQN_1 +HGG__BraTS19_CBICA_ASA_1 +HGG__BraTS19_CBICA_ASK_1 +HGG__BraTS19_CBICA_ASO_1 +HGG__BraTS19_CBICA_AWH_1 +HGG__BraTS19_CBICA_AWV_1 +HGG__BraTS19_CBICA_AYA_1 +HGG__BraTS19_CBICA_AYC_1 +HGG__BraTS19_CBICA_AYI_1 +HGG__BraTS19_CBICA_BFB_1 +HGG__BraTS19_CBICA_BGN_1 +HGG__BraTS19_CBICA_BGR_1 +HGG__BraTS19_CBICA_BJY_1 +HGG__BraTS19_TCIA01_231_1 +HGG__BraTS19_TCIA01_378_1 +HGG__BraTS19_TCIA01_390_1 +HGG__BraTS19_TCIA01_412_1 +HGG__BraTS19_TCIA02_135_1 +HGG__BraTS19_TCIA02_179_1 +HGG__BraTS19_TCIA02_208_1 +HGG__BraTS19_TCIA02_274_1 +HGG__BraTS19_TCIA02_314_1 +HGG__BraTS19_TCIA02_430_1 +HGG__BraTS19_TCIA02_608_1 +HGG__BraTS19_TCIA03_121_1 +HGG__BraTS19_TCIA03_138_1 +HGG__BraTS19_TCIA03_375_1 +HGG__BraTS19_TCIA03_498_1 +HGG__BraTS19_TCIA06_184_1 +HGG__BraTS19_TCIA06_372_1 +HGG__BraTS19_TCIA08_113_1 +HGG__BraTS19_TCIA08_162_1 +HGG__BraTS19_TCIA08_218_1 +HGG__BraTS19_TCIA08_469_1 +LGG__BraTS19_2013_6_1 +LGG__BraTS19_TCIA09_141_1 +LGG__BraTS19_TCIA09_255_1 +LGG__BraTS19_TCIA09_402_1 +LGG__BraTS19_TCIA09_451_1 +LGG__BraTS19_TCIA09_462_1 +LGG__BraTS19_TCIA09_620_1 +LGG__BraTS19_TCIA10_266_1 +LGG__BraTS19_TCIA10_413_1 +LGG__BraTS19_TCIA10_628_1 +LGG__BraTS19_TCIA10_629_1 +LGG__BraTS19_TCIA10_640_1 +LGG__BraTS19_TCIA12_298_1 +LGG__BraTS19_TCIA12_470_1 +LGG__BraTS19_TCIA13_621_1 +LGG__BraTS19_TCIA13_624_1 +LGG__BraTS19_TCIA13_654_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold1_validation.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold1_validation.txt new file mode 100644 index 00000000000..d24f39b67c4 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold1_validation.txt @@ -0,0 +1,67 @@ +HGG__BraTS19_2013_13_1 +HGG__BraTS19_2013_19_1 +HGG__BraTS19_2013_27_1 +HGG__BraTS19_CBICA_AAG_1 +HGG__BraTS19_CBICA_ALN_1 +HGG__BraTS19_CBICA_ANV_1 +HGG__BraTS19_CBICA_AOH_1 +HGG__BraTS19_CBICA_APK_1 +HGG__BraTS19_CBICA_APR_1 +HGG__BraTS19_CBICA_AQG_1 +HGG__BraTS19_CBICA_AQP_1 +HGG__BraTS19_CBICA_ARZ_1 +HGG__BraTS19_CBICA_ASF_1 +HGG__BraTS19_CBICA_ASG_1 +HGG__BraTS19_CBICA_ATP_1 +HGG__BraTS19_CBICA_ATX_1 +HGG__BraTS19_CBICA_AUA_1 +HGG__BraTS19_CBICA_AVJ_1 +HGG__BraTS19_CBICA_AVV_1 +HGG__BraTS19_CBICA_AWG_1 +HGG__BraTS19_CBICA_AXL_1 +HGG__BraTS19_CBICA_AXQ_1 +HGG__BraTS19_CBICA_BAN_1 +HGG__BraTS19_CBICA_BBG_1 +HGG__BraTS19_CBICA_BGE_1 +HGG__BraTS19_CBICA_BHQ_1 +HGG__BraTS19_CBICA_BIC_1 +HGG__BraTS19_CBICA_BNR_1 +HGG__BraTS19_TCIA01_131_1 +HGG__BraTS19_TCIA01_147_1 +HGG__BraTS19_TCIA01_180_1 +HGG__BraTS19_TCIA01_190_1 +HGG__BraTS19_TCIA01_221_1 +HGG__BraTS19_TCIA01_335_1 +HGG__BraTS19_TCIA01_411_1 +HGG__BraTS19_TCIA02_151_1 +HGG__BraTS19_TCIA02_321_1 +HGG__BraTS19_TCIA02_331_1 +HGG__BraTS19_TCIA02_368_1 +HGG__BraTS19_TCIA02_471_1 +HGG__BraTS19_TCIA03_257_1 +HGG__BraTS19_TCIA03_474_1 +HGG__BraTS19_TCIA04_111_1 +HGG__BraTS19_TCIA04_328_1 +HGG__BraTS19_TCIA04_343_1 +HGG__BraTS19_TCIA05_277_1 +HGG__BraTS19_TCIA05_478_1 +HGG__BraTS19_TCIA06_165_1 +HGG__BraTS19_TCIA08_105_1 +HGG__BraTS19_TCIA08_280_1 +HGG__BraTS19_TMC_15477_1 +HGG__BraTS19_TMC_21360_1 +HGG__BraTS19_TMC_30014_1 +LGG__BraTS19_TCIA09_428_1 +LGG__BraTS19_TCIA10_175_1 +LGG__BraTS19_TCIA10_276_1 +LGG__BraTS19_TCIA10_393_1 +LGG__BraTS19_TCIA10_408_1 +LGG__BraTS19_TCIA10_410_1 +LGG__BraTS19_TCIA10_449_1 +LGG__BraTS19_TCIA10_490_1 +LGG__BraTS19_TCIA10_625_1 +LGG__BraTS19_TCIA10_637_1 +LGG__BraTS19_TCIA12_249_1 +LGG__BraTS19_TCIA12_466_1 +LGG__BraTS19_TCIA13_615_1 +LGG__BraTS19_TCIA13_630_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold2_validation.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold2_validation.txt new file mode 100644 index 00000000000..c468e57417d --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold2_validation.txt @@ -0,0 +1,67 @@ +HGG__BraTS19_2013_11_1 +HGG__BraTS19_2013_21_1 +HGG__BraTS19_2013_2_1 +HGG__BraTS19_2013_4_1 +HGG__BraTS19_CBICA_ABB_1 +HGG__BraTS19_CBICA_ABE_1 +HGG__BraTS19_CBICA_ABM_1 +HGG__BraTS19_CBICA_ANZ_1 +HGG__BraTS19_CBICA_AOP_1 +HGG__BraTS19_CBICA_APY_1 +HGG__BraTS19_CBICA_AQA_1 +HGG__BraTS19_CBICA_AQO_1 +HGG__BraTS19_CBICA_AQU_1 +HGG__BraTS19_CBICA_ARW_1 +HGG__BraTS19_CBICA_ASV_1 +HGG__BraTS19_CBICA_AUN_1 +HGG__BraTS19_CBICA_AUW_1 +HGG__BraTS19_CBICA_AUX_1 +HGG__BraTS19_CBICA_AVB_1 +HGG__BraTS19_CBICA_AVF_1 +HGG__BraTS19_CBICA_AWX_1 +HGG__BraTS19_CBICA_AXO_1 +HGG__BraTS19_CBICA_AYW_1 +HGG__BraTS19_CBICA_BAX_1 +HGG__BraTS19_CBICA_BEM_1 +HGG__BraTS19_CBICA_BHK_1 +HGG__BraTS19_CBICA_BHM_1 +HGG__BraTS19_CBICA_BLJ_1 +HGG__BraTS19_TCIA01_150_1 +HGG__BraTS19_TCIA01_203_1 +HGG__BraTS19_TCIA01_235_1 +HGG__BraTS19_TCIA01_401_1 +HGG__BraTS19_TCIA01_448_1 +HGG__BraTS19_TCIA01_499_1 +HGG__BraTS19_TCIA02_168_1 +HGG__BraTS19_TCIA02_222_1 +HGG__BraTS19_TCIA02_226_1 +HGG__BraTS19_TCIA02_283_1 +HGG__BraTS19_TCIA02_290_1 +HGG__BraTS19_TCIA02_309_1 +HGG__BraTS19_TCIA02_394_1 +HGG__BraTS19_TCIA02_455_1 +HGG__BraTS19_TCIA02_606_1 +HGG__BraTS19_TCIA03_133_1 +HGG__BraTS19_TCIA04_192_1 +HGG__BraTS19_TCIA04_361_1 +HGG__BraTS19_TCIA06_332_1 +HGG__BraTS19_TCIA08_167_1 +HGG__BraTS19_TCIA08_205_1 +HGG__BraTS19_TCIA08_234_1 +HGG__BraTS19_TCIA08_242_1 +HGG__BraTS19_TCIA08_278_1 +HGG__BraTS19_TCIA08_436_1 +HGG__BraTS19_TMC_12866_1 +LGG__BraTS19_2013_15_1 +LGG__BraTS19_2013_1_1 +LGG__BraTS19_TCIA09_312_1 +LGG__BraTS19_TCIA10_109_1 +LGG__BraTS19_TCIA10_130_1 +LGG__BraTS19_TCIA10_152_1 +LGG__BraTS19_TCIA10_241_1 +LGG__BraTS19_TCIA10_282_1 +LGG__BraTS19_TCIA10_325_1 +LGG__BraTS19_TCIA10_639_1 +LGG__BraTS19_TCIA13_618_1 +LGG__BraTS19_TCIA13_633_1 +LGG__BraTS19_TMC_09043_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold3_validation.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold3_validation.txt new file mode 100644 index 00000000000..171a51a02a8 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold3_validation.txt @@ -0,0 +1,67 @@ +HGG__BraTS19_2013_12_1 +HGG__BraTS19_2013_14_1 +HGG__BraTS19_2013_18_1 +HGG__BraTS19_2013_20_1 +HGG__BraTS19_2013_26_1 +HGG__BraTS19_CBICA_ABO_1 +HGG__BraTS19_CBICA_ALX_1 +HGG__BraTS19_CBICA_ANP_1 +HGG__BraTS19_CBICA_AOS_1 +HGG__BraTS19_CBICA_AOZ_1 +HGG__BraTS19_CBICA_AQT_1 +HGG__BraTS19_CBICA_ARF_1 +HGG__BraTS19_CBICA_ASE_1 +HGG__BraTS19_CBICA_ASW_1 +HGG__BraTS19_CBICA_ATN_1 +HGG__BraTS19_CBICA_ATV_1 +HGG__BraTS19_CBICA_AUQ_1 +HGG__BraTS19_CBICA_AVG_1 +HGG__BraTS19_CBICA_AVT_1 +HGG__BraTS19_CBICA_AWI_1 +HGG__BraTS19_CBICA_AXW_1 +HGG__BraTS19_CBICA_AYG_1 +HGG__BraTS19_CBICA_AYU_1 +HGG__BraTS19_CBICA_BAP_1 +HGG__BraTS19_CBICA_BCL_1 +HGG__BraTS19_CBICA_BDK_1 +HGG__BraTS19_CBICA_BGG_1 +HGG__BraTS19_CBICA_BGT_1 +HGG__BraTS19_CBICA_BGW_1 +HGG__BraTS19_CBICA_BGX_1 +HGG__BraTS19_TCIA01_186_1 +HGG__BraTS19_TCIA01_429_1 +HGG__BraTS19_TCIA01_460_1 +HGG__BraTS19_TCIA02_171_1 +HGG__BraTS19_TCIA02_370_1 +HGG__BraTS19_TCIA02_374_1 +HGG__BraTS19_TCIA02_377_1 +HGG__BraTS19_TCIA02_473_1 +HGG__BraTS19_TCIA02_491_1 +HGG__BraTS19_TCIA02_607_1 +HGG__BraTS19_TCIA03_296_1 +HGG__BraTS19_TCIA03_338_1 +HGG__BraTS19_TCIA03_419_1 +HGG__BraTS19_TCIA04_437_1 +HGG__BraTS19_TCIA04_479_1 +HGG__BraTS19_TCIA06_247_1 +HGG__BraTS19_TCIA06_603_1 +HGG__BraTS19_TMC_11964_1 +LGG__BraTS19_2013_28_1 +LGG__BraTS19_2013_29_1 +LGG__BraTS19_2013_9_1 +LGG__BraTS19_TCIA09_177_1 +LGG__BraTS19_TCIA09_254_1 +LGG__BraTS19_TCIA10_103_1 +LGG__BraTS19_TCIA10_299_1 +LGG__BraTS19_TCIA10_310_1 +LGG__BraTS19_TCIA10_330_1 +LGG__BraTS19_TCIA10_346_1 +LGG__BraTS19_TCIA10_351_1 +LGG__BraTS19_TCIA10_420_1 +LGG__BraTS19_TCIA10_442_1 +LGG__BraTS19_TCIA10_632_1 +LGG__BraTS19_TCIA10_644_1 +LGG__BraTS19_TCIA12_480_1 +LGG__BraTS19_TCIA13_623_1 +LGG__BraTS19_TCIA13_642_1 +LGG__BraTS19_TCIA13_645_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold4_validation.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold4_validation.txt new file mode 100644 index 00000000000..0fc2a8bc9cc --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/folds/fold4_validation.txt @@ -0,0 +1,67 @@ +HGG__BraTS19_2013_10_1 +HGG__BraTS19_2013_17_1 +HGG__BraTS19_2013_25_1 +HGG__BraTS19_CBICA_AAP_1 +HGG__BraTS19_CBICA_ABY_1 +HGG__BraTS19_CBICA_AMH_1 +HGG__BraTS19_CBICA_ANI_1 +HGG__BraTS19_CBICA_AOO_1 +HGG__BraTS19_CBICA_AQQ_1 +HGG__BraTS19_CBICA_AQR_1 +HGG__BraTS19_CBICA_AQV_1 +HGG__BraTS19_CBICA_AQY_1 +HGG__BraTS19_CBICA_AQZ_1 +HGG__BraTS19_CBICA_ASH_1 +HGG__BraTS19_CBICA_ASN_1 +HGG__BraTS19_CBICA_ASR_1 +HGG__BraTS19_CBICA_ASU_1 +HGG__BraTS19_CBICA_ASY_1 +HGG__BraTS19_CBICA_ATB_1 +HGG__BraTS19_CBICA_ATD_1 +HGG__BraTS19_CBICA_ATF_1 +HGG__BraTS19_CBICA_AUR_1 +HGG__BraTS19_CBICA_AXJ_1 +HGG__BraTS19_CBICA_AXM_1 +HGG__BraTS19_CBICA_AXN_1 +HGG__BraTS19_CBICA_AZD_1 +HGG__BraTS19_CBICA_AZH_1 +HGG__BraTS19_CBICA_BCF_1 +HGG__BraTS19_CBICA_BFP_1 +HGG__BraTS19_CBICA_BGO_1 +HGG__BraTS19_CBICA_BHB_1 +HGG__BraTS19_CBICA_BHV_1 +HGG__BraTS19_CBICA_BHZ_1 +HGG__BraTS19_CBICA_BKV_1 +HGG__BraTS19_TCIA01_201_1 +HGG__BraTS19_TCIA01_425_1 +HGG__BraTS19_TCIA02_117_1 +HGG__BraTS19_TCIA02_118_1 +HGG__BraTS19_TCIA02_198_1 +HGG__BraTS19_TCIA02_300_1 +HGG__BraTS19_TCIA02_322_1 +HGG__BraTS19_TCIA02_605_1 +HGG__BraTS19_TCIA03_199_1 +HGG__BraTS19_TCIA03_265_1 +HGG__BraTS19_TCIA04_149_1 +HGG__BraTS19_TCIA05_396_1 +HGG__BraTS19_TCIA05_444_1 +HGG__BraTS19_TCIA06_211_1 +HGG__BraTS19_TCIA06_409_1 +HGG__BraTS19_TCIA08_319_1 +HGG__BraTS19_TCIA08_406_1 +HGG__BraTS19_TMC_06290_1 +HGG__BraTS19_TMC_06643_1 +HGG__BraTS19_TMC_27374_1 +LGG__BraTS19_2013_0_1 +LGG__BraTS19_2013_16_1 +LGG__BraTS19_2013_24_1 +LGG__BraTS19_2013_8_1 +LGG__BraTS19_TCIA09_493_1 +LGG__BraTS19_TCIA10_202_1 +LGG__BraTS19_TCIA10_261_1 +LGG__BraTS19_TCIA10_307_1 +LGG__BraTS19_TCIA10_387_1 +LGG__BraTS19_TCIA12_101_1 +LGG__BraTS19_TCIA13_634_1 +LGG__BraTS19_TCIA13_650_1 +LGG__BraTS19_TCIA13_653_1 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/mlperf.conf b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/mlperf.conf new file mode 100644 index 00000000000..7f5b55b58e2 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/mlperf.conf @@ -0,0 +1,65 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +mobilenet.*.performance_sample_count_override = 1024 +gnmt.*.performance_sample_count_override = 3903900 +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +ssd-resnet34.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +3d-unet.*.performance_sample_count_override = 16 + +# Set seeds. The seeds will be distributed two weeks before the submission. +*.*.qsl_rng_seed = 12786827339337101903 +*.*.sample_index_rng_seed = 12640797754436136668 +*.*.schedule_rng_seed = 3135815929913719677 + +*.SingleStream.target_latency_percentile = 90 +*.SingleStream.min_duration = 60000 +*.SingleStream.min_query_count = 1024 + +*.MultiStream.target_qps = 20 +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.max_async_queries = 1 +*.MultiStream.target_latency = 50 +*.MultiStream.min_duration = 60000 +*.MultiStream.min_query_count = 270336 +ssd-resnet34.MultiStream.target_qps = 15 +ssd-resnet34.MultiStream.target_latency = 66 +gnmt.MultiStream.min_query_count = 90112 +gnmt.MultiStream.target_latency = 100 +gnmt.MultiStream.target_qps = 10 +gnmt.MultiStream.target_latency_percentile = 97 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 60000 +*.Server.min_query_count = 270336 +resnet50.Server.target_latency = 15 +ssd-resnet34.Server.target_latency = 100 +gnmt.Server.min_query_count = 90112 +gnmt.Server.target_latency = 250 +gnmt.Server.target_latency_percentile = 97 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 30 +rnnt.Server.target_latency = 1000 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 60000 +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario, so set this to 24576 since +# the rule requires that Offline scenario run for at least 24576 samples. +*.Offline.min_query_count = 24576 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 +*.MultiStream.samples_per_query = 4 diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/onnxruntime_SUT.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/onnxruntime_SUT.py new file mode 100644 index 00000000000..0651e2fedd7 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/onnxruntime_SUT.py @@ -0,0 +1,61 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import json +import os +import sys +sys.path.insert(0, os.getcwd()) + +import mlperf_loadgen as lg +import numpy as np +import onnxruntime + +from brats_QSL import get_brats_QSL + +class _3DUNET_ONNXRuntime_SUT(): + def __init__(self, model_path, preprocessed_data_dir, performance_count): + print("Loading ONNX model...") + self.sess = onnxruntime.InferenceSession(model_path) + + print("Constructing SUT...") + self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries, self.process_latencies) + self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count) + print("Finished constructing SUT.") + + def issue_queries(self, query_samples): + for i in range(len(query_samples)): + data = self.qsl.get_features(query_samples[i].index) + + print("Processing sample id {:d} with shape = {:}".format(query_samples[i].index, data.shape)) + + # Follow the PyTorch implementation. + # The ONNX file has five outputs, but we only care about the one named "output". + output = self.sess.run(["output"], {"input": data[np.newaxis, ...]})[0].squeeze(0).astype(np.float16) + + response_array = array.array("B", output.tobytes()) + bi = response_array.buffer_info() + response = lg.QuerySampleResponse(query_samples[i].id, bi[0], bi[1]) + lg.QuerySamplesComplete([response]) + + def flush_queries(self): + pass + + def process_latencies(self, latencies_ns): + pass + +def get_onnxruntime_sut(model_path, preprocessed_data_dir, performance_count): + return _3DUNET_ONNXRuntime_SUT(model_path, preprocessed_data_dir, performance_count) \ No newline at end of file diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/ov_SUT.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/ov_SUT.py new file mode 100644 index 00000000000..7bc8411fc55 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/ov_SUT.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright (c) 2020 INTEL CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import json +import os +import sys + +sys.path.insert(0, os.getcwd()) + +import mlperf_loadgen as lg +import numpy as np + +from brats_QSL import get_brats_QSL + +from openvino.inference_engine import IECore +from scipy.special import softmax + + +class _3DUNET_OV_SUT(): + def __init__(self, model_path, preprocessed_data_dir, performance_count): + print("Loading OV model...") + + model_xml = model_path + model_bin = os.path.splitext(model_xml)[0] + '.bin' + + ie = IECore() + net = ie.read_network(model=model_xml, weights=model_bin) + + self.input_name = next(iter(net.inputs)) + + # After model conversion output name could be any + # So we are looking for output with max number of channels + max_channels = 0 + for output in net.outputs: + if max_channels < net.outputs[output].shape[-1]: + _3DUNET_OV_SUT.output_name = output + + self.exec_net = ie.load_network(network=net, device_name='CPU') + + print("Constructing SUT...") + self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries, + self.process_latencies) + self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count) + print("Finished constructing SUT.") + + def issue_queries(self, query_samples): + for i in range(len(query_samples)): + data = self.qsl.get_features(query_samples[i].index) + + print("Processing sample id {:d} with shape = {:}".format( + query_samples[i].index, data.shape)) + + output = self.exec_net.infer( + inputs={self.input_name: data[np.newaxis, ...]})[ + _3DUNET_OV_SUT.output_name].astype(np.float16) + + response_array = array.array("B", output.tobytes()) + bi = response_array.buffer_info() + response = lg.QuerySampleResponse(query_samples[i].id, bi[0], + bi[1]) + lg.QuerySamplesComplete([response]) + + def flush_queries(self): + pass + + def process_latencies(self, latencies_ns): + pass + + +def get_ov_sut(model_path, preprocessed_data_dir, performance_count): + return _3DUNET_OV_SUT(model_path, preprocessed_data_dir, performance_count) \ No newline at end of file diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/preprocess.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/preprocess.py new file mode 100644 index 00000000000..758d92488ea --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/preprocess.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import numpy +import os +import pickle +import sys +import torch + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) + +from batchgenerators.augmentations.utils import pad_nd_image +from batchgenerators.utilities.file_and_folder_operations import subfiles +from nnunet.training.model_restore import load_model_and_checkpoint_files +from nnunet.inference.predict import preprocess_multithreaded + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--model_dir", default="build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1", + help="Path to the directory containing plans.pkl") + parser.add_argument("--raw_data_dir", default="build/raw_data/nnUNet_raw_data/Task043_BraTS2019/imagesTr", + help="Path to the directory containing raw nii.gz files") + parser.add_argument("--preprocessed_data_dir", default="build/preprocessed_data", help="Path to the directory containing preprocessed data") + parser.add_argument("--validation_fold_file", default="folds/fold1_validation.txt", help="Path to the txt file storing all the sample names for the validation fold") + parser.add_argument("--num_threads_preprocessing", type=int, default=12, help="Number of threads to run the preprocessing with") + args = parser.parse_args() + return args + +def preprocess_MLPerf(model, checkpoint_name, folds, fp16, list_of_lists, output_filenames, preprocessing_folder, num_threads_preprocessing): + assert len(list_of_lists) == len(output_filenames) + + print("loading parameters for folds", folds) + trainer, params = load_model_and_checkpoint_files(model, folds, fp16=fp16, checkpoint_name=checkpoint_name) + + print("starting preprocessing generator") + preprocessing = preprocess_multithreaded(trainer, list_of_lists, output_filenames, num_threads_preprocessing, None) + print("Preprocessing images...") + all_output_files = [] + + for preprocessed in preprocessing: + output_filename, (d, dct) = preprocessed + + all_output_files.append(output_filename) + if isinstance(d, str): + data = np.load(d) + os.remove(d) + d = data + + # Pad to the desired full volume + d = pad_nd_image(d, trainer.patch_size, "constant", None, False, None) + + with open(os.path.join(preprocessing_folder, output_filename+ ".pkl"), "wb") as f: + pickle.dump([d, dct], f) + f.close() + + return all_output_files + + +def main(): + args = get_args() + + print("Preparing for preprocessing data...") + + # Validation set is fold 1 + fold = 1 + validation_fold_file = args.validation_fold_file + + # Make sure the model exists + model_dir = args.model_dir + model_path = os.path.join(model_dir, "plans.pkl") + assert os.path.isfile(model_path), "Cannot find the model file {:}!".format(model_path) + checkpoint_name = "model_final_checkpoint" + + # Other settings + fp16 = False + num_threads_preprocessing = args.num_threads_preprocessing + raw_data_dir = args.raw_data_dir + preprocessed_data_dir = args.preprocessed_data_dir + + # Open list containing validation images from specific fold (e.g. 1) + validation_files = [] + with open(validation_fold_file) as f: + for line in f: + validation_files.append(line.rstrip()) + + # Create output and preprocessed directory + if not os.path.isdir(preprocessed_data_dir): + os.makedirs(preprocessed_data_dir) + + # Create list of images locations (i.e. 4 images per case => 4 modalities) + all_files = subfiles(raw_data_dir, suffix=".nii.gz", join=False, sort=True) + list_of_lists = [[os.path.join(raw_data_dir, i) for i in all_files if i[:len(j)].startswith(j) and + len(i) == (len(j) + 12)] for j in validation_files] + + # Preprocess images, returns filenames list + # This runs in multiprocess + print("Acually preprocessing data...") + preprocessed_files = preprocess_MLPerf(model_dir, checkpoint_name, fold, fp16, list_of_lists, + validation_files, preprocessed_data_dir, num_threads_preprocessing) + + print("Saving metadata of the preprocessed data...") + with open(os.path.join(preprocessed_data_dir, "preprocessed_files.pkl"), "wb") as f: + pickle.dump(preprocessed_files, f) + + print("Preprocessed data saved to {:}".format(preprocessed_data_dir)) + print("Done!") + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/pytorch_SUT.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/pytorch_SUT.py new file mode 100644 index 00000000000..108a1778c6c --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/pytorch_SUT.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# Copyright (c) 2021 INTEL CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import json +import os +import sys +sys.path.insert(0, os.getcwd()) + +import mlperf_loadgen as lg +import numpy as np +import torch +import torch.nn.functional as F +from brats_QSL import get_brats_QSL + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) +from nnunet.training.model_restore import load_model_and_checkpoint_files + +class _3DUNET_PyTorch_SUT(): + def __init__(self, model, preprocessed_data_dir, performance_count, folds, checkpoint_name): + + print("Loading PyTorch model...") + #model_path = os.path.join(model_dir, "plans.pkl") + #assert os.path.isfile(model_path), "Cannot find the model file {:}!".format(model_path) + #self.trainer, params = load_model_and_checkpoint_files(model_dir, folds, fp16=False, checkpoint_name=checkpoint_name) + #self.trainer.load_checkpoint_ram(params[0], False) + self.model = model + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + print("Constructing SUT...") + self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries, self.process_latencies) + print("Finished constructing SUT.") + self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count) + + def issue_queries(self, query_samples): + with torch.no_grad(): + for i in range(len(query_samples)): + data = self.qsl.get_features(query_samples[i].index) + + print("Processing sample id {:d} with shape = {:}".format(query_samples[i].index, data.shape)) + + image = torch.from_numpy(data[np.newaxis,...]).float().to(self.device) + output = self.model(image)[0].cpu().numpy().astype(np.float16) + #output = self.trainer.network(image)[0].cpu().numpy().astype(np.float16) + + #transpose_forward = self.trainer.plans.get("transpose_forward") + #transpose_backward = self.trainer.plans.get("transpose_backward") + #assert transpose_forward == [0, 1, 2], "Unexpected transpose_forward {:}".format(transpose_forward) + #assert transpose_backward == [0, 1, 2], "Unexpected transpose_backward {:}".format(transpose_backward) + + response_array = array.array("B", output.tobytes()) + bi = response_array.buffer_info() + response = lg.QuerySampleResponse(query_samples[i].id, bi[0], bi[1]) + lg.QuerySamplesComplete([response]) + + def flush_queries(self): + pass + + def process_latencies(self, latencies_ns): + pass + +def get_pytorch_sut(model, preprocessed_data_dir, performance_count, folds=1, checkpoint_name="model_final_checkpoint"): + return _3DUNET_PyTorch_SUT(model, preprocessed_data_dir, performance_count, folds, checkpoint_name) diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/requirements.txt b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/requirements.txt new file mode 100644 index 00000000000..7a26fbe388d --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/requirements.txt @@ -0,0 +1,8 @@ +neural-compressor +numpy<=1.23.5 +batchgenerators==0.20.0 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch>=1.6.0+cpu +torchvision>=0.7.0+cpu +scipy<1.9.2, >=1.8 +scikit-learn diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run.py new file mode 100644 index 00000000000..b1591601faf --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020 - 2021 INTEL CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +sys.path.insert(0, os.getcwd()) +import time + +import argparse +import mlperf_loadgen as lg +import subprocess +import torch +import torch.nn as nn +import numpy as np +import torchvision.transforms as transforms +import torchvision.datasets as datasets + + +parser = argparse.ArgumentParser(description='PyTorch 3d-unet Training') +parser = argparse.ArgumentParser() +parser.add_argument("--backend", + choices=["pytorch", "onnxruntime", "tf", "ov"], + default="pytorch", + help="Backend") +parser.add_argument( + "--scenario", + choices=["SingleStream", "Offline", "Server", "MultiStream"], + default="Offline", + help="Scenario") +parser.add_argument("--accuracy", + action="store_true", + help="enable accuracy pass") +parser.add_argument("--mlperf_conf", + default="build/mlperf.conf", + help="mlperf rules config") +parser.add_argument("--user_conf", + default="user.conf", + help="user config for user LoadGen settings such as target QPS") +parser.add_argument( + "--model_dir", + default= + "build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1", + help="Path to the directory containing plans.pkl") +parser.add_argument("--model", help="Path to the ONNX, OpenVINO, or TF model") +parser.add_argument("--preprocessed_data_dir", + default="build/preprocessed_data", + help="path to preprocessed data") +parser.add_argument("--performance_count", + type=int, + default=16, + help="performance count") +parser.add_argument('--tune', dest='tune', action='store_true', + help='tune best int8 model on calibration dataset') +parser.add_argument('--performance', dest='performance', action='store_true', + help='run benchmark') +parser.add_argument('--int8', dest='int8', action='store_true', + help='run benchmark for int8') +parser.add_argument('-b', '--batch_size', default=8, type=int, + metavar='N', help='mini-batch size (default: 8)') +parser.add_argument('-i', "--iter", default=0, type=int, + help='For accuracy measurement only.') +parser.add_argument('-w', "--warmup_iter", default=5, type=int, + help='For benchmark measurement only.') +parser.add_argument("--tuned_checkpoint", default='./saved_results', type=str, metavar='PATH', + help='path to checkpoint tuned by Neural Compressor' + ' (default: ./)') +args = parser.parse_args() + + +scenario_map = { + "SingleStream": lg.TestScenario.SingleStream, + "Offline": lg.TestScenario.Offline, + "Server": lg.TestScenario.Server, + "MultiStream": lg.TestScenario.MultiStream +} + + +def eval_func(model): + if args.backend == "pytorch": + from pytorch_SUT import get_pytorch_sut + sut = get_pytorch_sut(model, args.preprocessed_data_dir, + args.performance_count) + elif args.backend == "onnxruntime": + from onnxruntime_SUT import get_onnxruntime_sut + sut = get_onnxruntime_sut(args.model, args.preprocessed_data_dir, + args.performance_count) + elif args.backend == "tf": + from tf_SUT import get_tf_sut + sut = get_tf_sut(args.model, args.preprocessed_data_dir, + args.performance_count) + elif args.backend == "ov": + from ov_SUT import get_ov_sut + sut = get_ov_sut(args.model, args.preprocessed_data_dir, + args.performance_count) + else: + raise ValueError("Unknown backend: {:}".format(args.backend)) + + settings = lg.TestSettings() + settings.scenario = scenario_map[args.scenario] + settings.FromConfig(args.mlperf_conf, "3d-unet", args.scenario) + settings.FromConfig(args.user_conf, "3d-unet", args.scenario) + + if args.accuracy: + settings.mode = lg.TestMode.AccuracyOnly + + log_path = "build/logs" + if not os.path.exists(log_path): + os.makedirs(log_path) + log_output_settings = lg.LogOutputSettings() + log_output_settings.outdir = log_path + log_output_settings.copy_summary_to_stdout = True + log_settings = lg.LogSettings() + log_settings.log_output = log_output_settings + + print("Running Loadgen test...") + lg.StartTestWithLogSettings(sut.sut, sut.qsl.qsl, settings, log_settings) + + if args.accuracy: + print("Running accuracy script...") + process = subprocess.Popen(['python3', 'accuracy-brats.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + + print(out) + print("Batch size = ", args.batch_size) + print("Accuracy: ", float(err)) + + lg.DestroySUT(sut.sut) + lg.DestroyQSL(sut.qsl.qsl) + return float(err) + + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) +from nnunet.training.model_restore import load_model_and_checkpoint_files +from neural_compressor.experimental import Quantization, common +import pickle + + +def main(): + args = parser.parse_args() + + class CalibrationDL(): + def __init__(self): + path = os.path.abspath(os.path.expanduser('./brats_cal_images_list.txt')) + with open(path, 'r') as f: + self.preprocess_files = [line.rstrip() for line in f] + + self.loaded_files = {} + self.batch_size = 1 + + def __getitem__(self, sample_id): + file_name = self.preprocess_files[sample_id] + print("Loading file {:}".format(file_name)) + with open(os.path.join('build/calib_preprocess/', "{:}.pkl".format(file_name)), "rb") as f: + self.loaded_files[sample_id] = pickle.load(f)[0] + # note that calibration phase does not care label, here we return 0 for label free case. + return self.loaded_files[sample_id], 0 + + def __len__(self): + self.count = len(self.preprocess_files) + return self.count + + + assert args.backend == "pytorch" + model_path = os.path.join(args.model_dir, "plans.pkl") + assert os.path.isfile(model_path), "Cannot find the model file {:}!".format(model_path) + trainer, params = load_model_and_checkpoint_files(args.model_dir, folds=1, fp16=False + , checkpoint_name='model_final_checkpoint') + trainer.load_checkpoint_ram(params[0], False) + model = trainer.network + + from neural_compressor.data import dataloaders + dataloader = dataloaders.DataLoader(dataset=CalibrationDL(), framework="pytorch_fx") + + if args.tune: + from neural_compressor import PostTrainingQuantConfig + from neural_compressor import quantization + conf = PostTrainingQuantConfig() + q_model = quantization.fit(model, + conf, + calib_dataloader=dataloader, + eval_func=eval_func) + q_model.save(args.tuned_checkpoint) + return + + if args.performance or args.accuracy: + model.eval() + if args.int8: + from neural_compressor.utils.pytorch import load + new_model = load(os.path.abspath(os.path.expanduser(args.tuned_checkpoint)), + model, + dataloader=dataloader) + else: + new_model = model + if args.performance: + from neural_compressor.config import BenchmarkConfig + from neural_compressor import benchmark + b_conf = BenchmarkConfig(warmup=5, + iteration=args.iter, + cores_per_instance=4, + num_of_instance=1) + benchmark.fit(new_model, b_conf, b_dataloader=dataloader) + if args.accuracy: + eval_func(new_model) + return + + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_benchmark.sh b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_benchmark.sh new file mode 100644 index 00000000000..cc163b3cdb0 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_benchmark.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -x + +function main { + + init_params "$@" + run_benchmark + +} + +# init params +function init_params { + iters=100 + tuned_checkpoint=saved_results + batch_size=8 + for var in "$@" + do + case $var in + --topology=*) + topology=$(echo $var |cut -f2 -d=) + ;; + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) + ;; + --input_model=*) + input_model=$(echo $var |cut -f2 -d=) + ;; + --mode=*) + mode=$(echo $var |cut -f2 -d=) + ;; + --batch_size=*) + batch_size=$(echo $var |cut -f2 -d=) + ;; + --iters=*) + iters=$(echo ${var} |cut -f2 -d=) + ;; + --int8=*) + int8=$(echo ${var} |cut -f2 -d=) + ;; + --config=*) + tuned_checkpoint=$(echo $var |cut -f2 -d=) + ;; + *) + echo "Error: No such parameter: ${var}" + exit 1 + ;; + esac + done + +} + + +# run_benchmark +function run_benchmark { + if [[ ${int8} == "true" ]]; then + extra_cmd="--int8" + else + extra_cmd="" + fi + + python run.py \ + --model_dir=${input_model} \ + --backend=pytorch --accuracy --performance \ + --preprocessed_data_dir=${dataset_location} \ + --mlperf_conf=./mlperf.conf \ + ${extra_cmd} +} + +main "$@" diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_tuning.sh b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_tuning.sh new file mode 100644 index 00000000000..50c39c41716 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/run_tuning.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -x + +function main { + + init_params "$@" + run_tuning + +} + +# init params +function init_params { + tuned_checkpoint=saved_results + for var in "$@" + do + case $var in + --topology=*) + topology=$(echo $var |cut -f2 -d=) + ;; + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) + ;; + --input_model=*) + input_model=$(echo $var |cut -f2 -d=) + ;; + --output_model=*) + tuned_checkpoint=$(echo $var |cut -f2 -d=) + ;; + *) + echo "Error: No such parameter: ${var}" + exit 1 + ;; + esac + done + +} + +# run_tuning +function run_tuning { + python run.py \ + --model_dir=${input_model} \ + --backend=pytorch --accuracy \ + --preprocessed_data_dir=${dataset_location} \ + --mlperf_conf=./mlperf.conf \ + --tune + +} + +main "$@" diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/tf_SUT.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/tf_SUT.py new file mode 100644 index 00000000000..866c0c21cdc --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/tf_SUT.py @@ -0,0 +1,73 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import json +import os +import sys +sys.path.insert(0, os.getcwd()) + +import mlperf_loadgen as lg +import numpy as np +import tensorflow as tf +from tensorflow.core.framework import graph_pb2 + +from brats_QSL import get_brats_QSL + + +class _3DUNET_TF_SUT(): + def __init__(self, model_path, preprocessed_data_dir, performance_count): + print("Loading TF model...") + graph_def = graph_pb2.GraphDef() + print(model_path) + with open(model_path, "rb") as f: + graph_def.ParseFromString(f.read()) + with tf.Graph().as_default() as g: + tf.compat.v1.import_graph_def(graph_def) + self.sess = tf.compat.v1.Session(graph=g) + self.input = g.get_tensor_by_name("import/input:0") + self.output = g.get_tensor_by_name("import/output:0") + + print("Constructing SUT...") + self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries, + self.process_latencies) + self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count) + print("Finished constructing SUT.") + + def issue_queries(self, query_samples): + for i in range(len(query_samples)): + data = self.qsl.get_features(query_samples[i].index) + + print("Processing sample id {:d} with shape = {:}".format( + query_samples[i].index, data.shape)) + + output = self.sess.run(self.output, feed_dict={self.input: data[np.newaxis, ...]})[0].astype(np.float16) + + response_array = array.array("B", output.tobytes()) + bi = response_array.buffer_info() + response = lg.QuerySampleResponse(query_samples[i].id, bi[0], + bi[1]) + lg.QuerySamplesComplete([response]) + + def flush_queries(self): + pass + + def process_latencies(self, latencies_ns): + pass + + +def get_tf_sut(model_path, preprocessed_data_dir, performance_count): + return _3DUNET_TF_SUT(model_path, preprocessed_data_dir, performance_count) \ No newline at end of file diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_onnx_to_tf.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_onnx_to_tf.py new file mode 100644 index 00000000000..472bcd61b1a --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_onnx_to_tf.py @@ -0,0 +1,59 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +sys.path.insert(0, os.getcwd()) + +import argparse +import onnx +import onnx_tf + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--onnx_model", + default="build/model/224_224_160.onnx", + help="Path to the ONNX model") + parser.add_argument("--output_name", + default="224_224_160.pb", + help="Name of output model") + parser.add_argument("--output_dir", + default="build/model", + help="Directory to save output model") + args = parser.parse_args() + return args + +def main(): + args = get_args() + + print("Loading ONNX model...") + onnx_model = onnx.load(args.onnx_model) + + print("Converting ONNX model to TF...") + tf_model = onnx_tf.backend.prepare(onnx_model) + + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) + + output_path = "./{}/{}".format(args.output_dir, args.output_name) + + tf_model.export_graph(output_path) + + print("Successfully exported model {}".format(output_path)) + + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_pytorch_to_onnx.py b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_pytorch_to_onnx.py new file mode 100644 index 00000000000..9652d1a6df9 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/unet_pytorch_to_onnx.py @@ -0,0 +1,76 @@ +# coding=utf-8 +# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved. +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +sys.path.insert(0, os.getcwd()) + +import argparse +import onnx +import torch + +sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet")) +from nnunet.training.model_restore import load_model_and_checkpoint_files + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--model_dir", + default="build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1", + help="Path to the PyTorch model") + parser.add_argument("--output_name", + default="224_224_160.onnx", + help="Name of output model") + parser.add_argument("--dynamic_bs_output_name", + default="224_224_160_dyanmic_bs.onnx", + help="Name of output model") + parser.add_argument("--output_dir", + default="build/model", + help="Directory to save output model") + args = parser.parse_args() + return args + +def main(): + args = get_args() + + print("Converting PyTorch model to ONNX...") + + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) + + output_path = "./{}/{}".format(args.output_dir, args.output_name) + dynamic_bs_output_path = "./{}/{}".format(args.output_dir, args.dynamic_bs_output_name) + + print("Loading Pytorch model...") + checkpoint_name = "model_final_checkpoint" + folds = 1 + trainer, params = load_model_and_checkpoint_files(args.model_dir, folds, fp16=False, checkpoint_name=checkpoint_name) + trainer.load_checkpoint_ram(params[0], False) + height = 224 + width = 224 + depth = 160 + channels = 4 + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + dummy_input = torch.rand([1, channels, height, width, depth]).float().to(device) + torch.onnx.export(trainer.network, dummy_input, output_path, opset_version=11, + input_names=['input'], output_names=['output']) + torch.onnx.export(trainer.network, dummy_input, dynamic_bs_output_path, opset_version=11, + input_names=['input'], output_names=['output'], + dynamic_axes=({"input": {0: "batch_size"}, "output": {0: "batch_size"}})) + + print("Successfully exported model {} and {}".format(output_path, dynamic_bs_output_path)) + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/user.conf b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/user.conf new file mode 100644 index 00000000000..545569c1ac6 --- /dev/null +++ b/examples/pytorch/image_recognition/3d-unet/quantization/ptq/fx/user.conf @@ -0,0 +1,6 @@ +# Please set these fields depending on the performance of your system to +# override default LoadGen settings. +*.SingleStream.target_latency = 10 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 +*.MultiStream.samples_per_query = 4 \ No newline at end of file