merge master

Signed-off-by: sdp <[email protected]>

.azure-pipelines/docker/Dockerfile.devel

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
 
 RUN ln -sf $(which python3) /usr/bin/python
 
-RUN python -m pip --no-cache-dir install --upgrade pip
+RUN python -m pip install pip==24.0
 RUN python -m pip install --no-cache-dir setuptools
 
 RUN pip list

.azure-pipelines/model-test.yml

@@ -57,7 +57,7 @@ parameters:
   - name: PyTorchModelList
     type: object
     default:
-      - resnet18
+#      - resnet18
       - resnet18_fx
   - name: ONNXModelList
     type: object

.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh

@@ -17,11 +17,6 @@ log_dir="$work_dir/../scanLog"
 mkdir -p $log_dir
 
 scan_path="scan_path.txt"
-if [ "${scan_module}" = "neural_solution" ]; then
-    scan_path="scan_path_neural_solution.txt"
-elif [ "${scan_module}" = "neural_insights" ]; then
-    scan_path="scan_path_neural_insights.txt"
-fi
 
 exit_code=0
 for line in $(cat ${work_dir}/${scan_path})

.azure-pipelines/scripts/codeScan/pylint/pylint.sh

@@ -44,22 +44,6 @@ pip install torch \
             pydantic \
             protobuf
 
-if [ "${scan_module}" = "neural_solution" ]; then
-    cd /neural-compressor
-    python setup.py install
-
-    echo "Install Neural Solution ... "
-    bash /neural-compressor/.azure-pipelines/scripts/install_neural_solution.sh
-
-elif [ "${scan_module}" = "neural_insights" ]; then
-    cd /neural-compressor
-    python setup.py install
-
-    echo "Install Neural Insights ... "
-    bash /neural-compressor/.azure-pipelines/scripts/install_neural_insights.sh
-
-fi
-
 echo "[DEBUG] list pipdeptree..."
 pip install pipdeptree
 pipdeptree

.azure-pipelines/scripts/install_nc.sh

@@ -2,10 +2,10 @@
 
 echo -e "\n Install Neural Compressor ... "
 cd /neural-compressor
-if [[ $1 = *"3x_pt" ]]; then
+if [[ $1 = *"3x_pt"* ]]; then
     python -m pip install --no-cache-dir -r requirements_pt.txt
     python setup.py pt bdist_wheel
-    pip install dist/neural_compressor*.whl --force-reinstall
+    pip install --no-deps dist/neural_compressor*.whl --force-reinstall
 elif [[ $1 = *"3x_tf"* ]]; then
     python -m pip install --no-cache-dir -r requirements_tf.txt
     python setup.py tf bdist_wheel

.azure-pipelines/scripts/ut/3x/collect_log_3x.sh

@@ -25,7 +25,8 @@ git config --global --add safe.directory /neural-compressor
 git fetch
 git checkout master
 rm -rf build dist *egg-info
-echo y | pip uninstall neural_compressor_${1}
+binary_index="${1%_fp8}"
+echo y | pip uninstall neural_compressor_${binary_index}
 cd /neural-compressor/.azure-pipelines-pr/scripts && bash install_nc.sh ${1}
 
 coverage erase

.azure-pipelines/scripts/ut/3x/coverage.3x_pt_fp8

@@ -0,0 +1,15 @@
+[run]
+branch = True
+
+[report]
+include =
+ */neural_compressor/torch/algorithms/habana_fp8/*
+ */neural_compressor/torch/amp/*
+exclude_lines =
+ pragma: no cover
+ raise NotImplementedError
+ raise TypeError
+ if self.device == "gpu":
+ if device == "gpu":
+ except ImportError:
+ except Exception as e:

.azure-pipelines/scripts/ut/3x/run_3x_pt.sh

@@ -15,6 +15,7 @@ inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__
 cd /neural-compressor/test/3x || exit 1
 rm -rf tensorflow
 rm -rf onnxrt
+rm -rf torch/algorithms/fp8_quant
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}

.azure-pipelines/scripts/ut/3x/run_3x_pt_fp8.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+python -c "import neural_compressor as nc"
+test_case="run 3x Torch Habana FP8"
+echo "${test_case}"
+
+# install requirements
+echo "set up UT env..."
+sed -i '/^intel_extension_for_pytorch/d' /neural-compressor/test/3x/torch/requirements.txt
+pip install -r /neural-compressor/test/3x/torch/requirements.txt
+pip install git+https://github.com/HabanaAI/[email protected]
+pip install pytest-cov
+pip install pytest-html
+pip list
+
+export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_pt_fp8
+inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
+cd /neural-compressor/test/3x || exit 1
+
+LOG_DIR=/neural-compressor/log_dir
+mkdir -p ${LOG_DIR}
+ut_log_name=${LOG_DIR}/ut_3x_pt_fp8.log
+pytest --cov="${inc_path}" -vs --disable-warnings --html=report.html --self-contained-html torch/algorithms/fp8_quant 2>&1 | tee -a ${ut_log_name}
+
+cp report.html ${LOG_DIR}/
+
+if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
+    echo "Find errors in pytest case, please check the output..."
+    echo "Please search for '== FAILURES ==' or '== ERRORS =='"
+    exit 1
+fi
+
+# if ut pass, collect the coverage file into artifacts
+cp .coverage ${LOG_DIR}/.coverage
+
+echo "UT finished successfully! "

.azure-pipelines/scripts/ut/run_basic_others.sh

@@ -14,7 +14,6 @@ cd /neural-compressor/test || exit 1
 find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
 sed -i '/ adaptor\//d' run.sh
 sed -i '/ tfnewapi\//d' run.sh
-sed -i '/ neural_coder\//d' run.sh
 sed -i '/ itex\//d' run.sh
 sed -i '/ pruning_with_pt/d' run.sh
 sed -i '/ pruning_with_tf/d' run.sh