Skip to content

Commit

Permalink
merge master
Browse files Browse the repository at this point in the history
Signed-off-by: zehao-intel <[email protected]>
  • Loading branch information
zehao-intel committed Jul 18, 2024
2 parents ae6651a + c8e4f40 commit 53b9930
Show file tree
Hide file tree
Showing 945 changed files with 4,926 additions and 147,415 deletions.
59 changes: 0 additions & 59 deletions .azure-pipelines/code-scan-neural-insights.yaml

This file was deleted.

59 changes: 0 additions & 59 deletions .azure-pipelines/code-scan-neural-solution.yaml

This file was deleted.

2 changes: 1 addition & 1 deletion .azure-pipelines/docker/Dockerfile.devel
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \

RUN ln -sf $(which python3) /usr/bin/python

RUN python -m pip --no-cache-dir install --upgrade pip
RUN python -m pip install pip==24.0
RUN python -m pip install --no-cache-dir setuptools

RUN pip list
Expand Down
2 changes: 1 addition & 1 deletion .azure-pipelines/model-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ parameters:
- name: PyTorchModelList
type: object
default:
- resnet18
# - resnet18
- resnet18_fx
- name: ONNXModelList
type: object
Expand Down
5 changes: 0 additions & 5 deletions .azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ log_dir="$work_dir/../scanLog"
mkdir -p $log_dir

scan_path="scan_path.txt"
if [ "${scan_module}" = "neural_solution" ]; then
scan_path="scan_path_neural_solution.txt"
elif [ "${scan_module}" = "neural_insights" ]; then
scan_path="scan_path_neural_insights.txt"
fi

exit_code=0
for line in $(cat ${work_dir}/${scan_path})
Expand Down

This file was deleted.

This file was deleted.

17 changes: 1 addition & 16 deletions .azure-pipelines/scripts/codeScan/pylint/pylint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ apt-get install -y --no-install-recommends --fix-missing \
build-essential

pip install -r /neural-compressor/requirements.txt
pip install -r /neural-compressor/requirements_pt.txt
pip install cmake

pip install torch \
Expand All @@ -44,22 +45,6 @@ pip install torch \
pydantic \
protobuf

if [ "${scan_module}" = "neural_solution" ]; then
cd /neural-compressor
python setup.py install

echo "Install Neural Solution ... "
bash /neural-compressor/.azure-pipelines/scripts/install_neural_solution.sh

elif [ "${scan_module}" = "neural_insights" ]; then
cd /neural-compressor
python setup.py install

echo "Install Neural Insights ... "
bash /neural-compressor/.azure-pipelines/scripts/install_neural_insights.sh

fi

echo "[DEBUG] list pipdeptree..."
pip install pipdeptree
pipdeptree
Expand Down
10 changes: 3 additions & 7 deletions .azure-pipelines/scripts/install_nc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,17 @@

echo -e "\n Install Neural Compressor ... "
cd /neural-compressor
if [[ $1 = *"3x_pt" ]]; then
if [[ $1 = *"3x_pt"* ]]; then
python -m pip install --no-cache-dir -r requirements_pt.txt
python setup.py pt bdist_wheel
pip install dist/neural_compressor*.whl --force-reinstall
pip install --no-deps dist/neural_compressor*.whl --force-reinstall
elif [[ $1 = *"3x_tf"* ]]; then
python -m pip install --no-cache-dir -r requirements_tf.txt
python setup.py tf bdist_wheel
pip install dist/neural_compressor*.whl --force-reinstall
elif [[ $1 = *"3x_ort" ]]; then
python -m pip install --no-cache-dir -r requirements_ort.txt
python setup.py ort bdist_wheel
pip install dist/neural_compressor*.whl --force-reinstall
else
python -m pip install --no-cache-dir -r requirements.txt
python setup.py 2x bdist_wheel
python setup.py bdist_wheel
pip install dist/neural_compressor*.whl --force-reinstall
fi

Expand Down
8 changes: 0 additions & 8 deletions .azure-pipelines/scripts/install_neural_insights.sh

This file was deleted.

27 changes: 0 additions & 27 deletions .azure-pipelines/scripts/install_neural_solution.sh

This file was deleted.

3 changes: 2 additions & 1 deletion .azure-pipelines/scripts/ut/3x/collect_log_3x.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ git config --global --add safe.directory /neural-compressor
git fetch
git checkout master
rm -rf build dist *egg-info
echo y | pip uninstall neural_compressor_${1}
binary_index="${1%_fp8}"
echo y | pip uninstall neural_compressor_${binary_index}
cd /neural-compressor/.azure-pipelines-pr/scripts && bash install_nc.sh ${1}

coverage erase
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ branch = True

[report]
include =
*/neural_compressor/common/*
*/neural_compressor/onnxrt/*
*/neural_compressor/torch/algorithms/habana_fp8/*
*/neural_compressor/torch/amp/*
exclude_lines =
pragma: no cover
raise NotImplementedError
Expand Down
1 change: 1 addition & 0 deletions .azure-pipelines/scripts/ut/3x/run_3x_pt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__
cd /neural-compressor/test/3x || exit 1
rm -rf tensorflow
rm -rf onnxrt
rm -rf torch/algorithms/fp8_quant

LOG_DIR=/neural-compressor/log_dir
mkdir -p ${LOG_DIR}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
#!/bin/bash
python -c "import neural_compressor as nc"
test_case="run 3x ONNXRT"
test_case="run 3x Torch Habana FP8"
echo "${test_case}"

# install requirements
echo "set up UT env..."
pip install -r /neural-compressor/test/3x/onnxrt/requirements.txt
sed -i '/^intel_extension_for_pytorch/d' /neural-compressor/test/3x/torch/requirements.txt
pip install -r /neural-compressor/test/3x/torch/requirements.txt
pip install git+https://github.com/HabanaAI/[email protected]
pip install pytest-cov
pip install pytest-html
pip list

export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_ort
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_pt_fp8
inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
cd /neural-compressor/test/3x || exit 1
rm -rf torch
rm -rf tensorflow

LOG_DIR=/neural-compressor/log_dir
mkdir -p ${LOG_DIR}
ut_log_name=${LOG_DIR}/ut_3x_ort.log
pytest --cov="${inc_path}" -vs --disable-warnings --html=report.html --self-contained-html . 2>&1 | tee -a ${ut_log_name}
ut_log_name=${LOG_DIR}/ut_3x_pt_fp8.log
pytest --cov="${inc_path}" -vs --disable-warnings --html=report.html --self-contained-html torch/algorithms/fp8_quant 2>&1 | tee -a ${ut_log_name}

cp report.html ${LOG_DIR}/

Expand Down
2 changes: 1 addition & 1 deletion .azure-pipelines/scripts/ut/env_setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ elif [[ $(echo "${test_case}" | grep -c "tf pruning") != 0 ]]; then
fi

if [[ $(echo "${test_case}" | grep -c "api") != 0 ]] || [[ $(echo "${test_case}" | grep -c "adaptor") != 0 ]]; then
pip install auto-round
pip install git+https://github.com/intel/auto-round.git@24b2e74070f2b4e6f26ff069ec75af74cf5b177c
fi

# test deps
Expand Down
1 change: 0 additions & 1 deletion .azure-pipelines/scripts/ut/run_basic_others.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ cd /neural-compressor/test || exit 1
find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
sed -i '/ adaptor\//d' run.sh
sed -i '/ tfnewapi\//d' run.sh
sed -i '/ neural_coder\//d' run.sh
sed -i '/ itex\//d' run.sh
sed -i '/ pruning_with_pt/d' run.sh
sed -i '/ pruning_with_tf/d' run.sh
Expand Down
3 changes: 3 additions & 0 deletions .azure-pipelines/scripts/ut/run_itrex.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ source /neural-compressor/.azure-pipelines/scripts/change_color.sh
python -c "import neural_compressor as nc;print(nc.version.__version__)"
echo "run itrex ut..."

# install inc 3x deps
pip install -r /neural-compressor/requirements_pt.txt

# prepare itrex
git clone https://github.com/intel/intel-extension-for-transformers.git /intel-extension-for-transformers
cd /intel-extension-for-transformers && git rev-parse --short HEAD
Expand Down
26 changes: 0 additions & 26 deletions .azure-pipelines/scripts/ut/run_ncoder.sh

This file was deleted.

Loading

0 comments on commit 53b9930

Please sign in to comment.