add code scan CI for neural insights (#996)

Signed-off-by: chensuyue <[email protected]>
Co-authored-by: bmyrcha <[email protected]>

.azure-pipelines/code-scan-neural-insights.yaml

@@ -0,0 +1,112 @@
+trigger: none
+
+pr:
+  autoCancel: true
+  drafts: false
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - neural_insights
+      - setup.py
+      - .azure-pipelines/code-scan-neural-insights.yml
+      - .azure-pipelines/scripts/codeScan
+
+pool:
+  vmImage: "ubuntu-latest"
+
+variables:
+  CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"
+
+stages:
+  - stage: DocStyleCodeScan
+    displayName: DocStyle Code Scan
+    dependsOn: []
+    jobs:
+      - job: DocStyle
+        displayName: DocStyle
+        steps:
+          - template: template/code-scan-template.yml
+            parameters:
+              codeScanFileName: "pydocstyle"
+              uploadPath: "pydocstyle.log"
+              scanModule: "neural_insights"
+
+  - stage: BanditCodeScan
+    displayName: Bandit Code Scan
+    dependsOn: []
+    jobs:
+      - job: Bandit
+        displayName: Bandit
+        steps:
+          - template: template/code-scan-template.yml
+            parameters:
+              codeScanFileName: "bandit"
+              uploadPath: "bandit.log"
+              scanModule: "neural_insights"
+
+  - stage: PylintCodeScan
+    displayName: Pylint Code Scan
+    dependsOn: []
+    jobs:
+      - job: Pylint
+        displayName: Pylint
+        steps:
+          - template: template/code-scan-template.yml
+            parameters:
+              codeScanFileName: "pylint"
+              uploadPath: "pylint.json"
+              scanModule: "neural_insights"
+
+  - stage: CopyRight
+    displayName: CopyRight Code Scan
+    dependsOn: []
+    jobs:
+      - job: CopyRight
+        displayName: CopyRight
+        steps:
+          - script: |
+              rm -fr $(Build.SourcesDirectory) || true
+              echo y | docker system prune
+            displayName: "Clean workspace"
+
+          - checkout: self
+            displayName: "Checkout out Repo"
+
+          - task: Bash@3
+            inputs:
+              targetType: "inline"
+              script: |
+                source $(Build.SourcesDirectory)/.azure-pipelines/scripts/change_color.sh
+                set -e
+                mkdir -p $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)
+                RESET="echo -en \\E[0m \\n"  # close 
+
+                supported_extensions=(py, sh, yaml)
+                git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(Build.SourcesDirectory)/neural_insights > $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log
+                files=$(cat $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++')
+
+                for file in ${files}
+                do
+                    if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
+                        if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
+                            echo ${file} >>  $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+                            $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
+                            cat   $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+                            $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
+                            $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1
+                        fi
+                    else
+                        $LIGHT_PURPLE && echo "Skipping ${file}"  && $RESET
+                    fi
+                done
+            displayName: "CopyRight Check"
+
+          - task: PublishPipelineArtifact@1
+            condition: failed()
+            inputs:
+              targetPath: $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+              artifact: copyright
+              publishLocation: "pipeline"
+            displayName: "PublishPipelineArtifact"

.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh

@@ -19,6 +19,8 @@ mkdir -p $log_dir
 scan_path="scan_path.txt"
 if [ "${scan_module}" = "neural_solution" ]; then
     scan_path="scan_path_neural_solution.txt"
+elif [ "${scan_module}" = "neural_insights" ]; then
+    scan_path="scan_path_neural_insights.txt"
 fi
 
 exit_code=0

.azure-pipelines/scripts/codeScan/pydocstyle/scan_path_neural_insights.txt

@@ -0,0 +1,4 @@
+/neural-compressor/neural_insights/components
+/neural-compressor/neural_insights/utils
+/neural-compressor/neural_insights/web
+

.azure-pipelines/scripts/codeScan/pylint/pylint.sh

@@ -39,6 +39,14 @@ if [ "${scan_module}" = "neural_solution" ]; then
 
     echo "Install Neural Solution ... "
     bash /neural-compressor/.azure-pipelines/scripts/install_neural_solution.sh
+
+elif [ "${scan_module}" = "neural_insights" ]; then
+    cd /neural-compressor
+    python setup.py install
+
+    echo "Install Neural Insights ... "
+    bash /neural-compressor/.azure-pipelines/scripts/install_neural_insights.sh
+
 fi
 
 python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \

.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml

@@ -15,4 +15,5 @@ matrix:
       - ${REPO_DIR}/neural_coder/*.md
       - ${REPO_DIR}/neural_solution/*.md
       - ${REPO_DIR}/neural_solution/docs/source/*.md
-      - ${REPO_DIR}/neural_solution/examples/**/*.md
+      - ${REPO_DIR}/neural_solution/examples/**/*.md
+      - ${REPO_DIR}/neural_insights/*.md

neural_insights/components/diagnosis/diagnosis.py

@@ -215,8 +215,7 @@ def get_histogram_data(self, op_name: str, inspect_type: str) -> list:
 
     @staticmethod
     def mse_metric_gap(fp32_tensor: Any, dequantize_tensor: Any) -> float:
-        """
-        Calculate the euclidean distance between fp32 tensor and int8 dequantize tensor.
+        """Calculate the euclidean distance between fp32 tensor and int8 dequantize tensor.
 
         Args:
             fp32_tensor (tensor): The FP32 tensor.
@@ -266,7 +265,7 @@ def get_weights_data(self, op_name: str, channel_normalization=True) -> list:
             if tensor_data_raw.ndim != 4:
                 continue
             tensor_data = tensor_data_raw[0]
-            shapes_order = self.model.shape_elements_order
+            shapes_order = self.model.shape_elements_order  # pylint: disable=no-member
             channels_index = shapes_order.index("channels")
             new_order = [channels_index]
             new_order.extend([x for x in range(len(shapes_order)) if x != channels_index])

neural_insights/components/model/repository.py

@@ -58,8 +58,7 @@ def get_frameworks(self) -> List[str]:
 
     @staticmethod
     def get_framework_from_path(model_path: str) -> str:
-        """
-        Get framework name from model extension.
+        """Get framework name from model extension.
 
         :param model_path: Path to model.
         """

neural_insights/components/workload_manager/quantization_workload.py

@@ -1,10 +1,26 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The quantization_workload module for Neural Insights quantization workloads."""
 from typing import Optional, Dict, Any
 
 from neural_insights.components.workload_manager.workload import Workload
 from neural_insights.utils.json_serializer import JsonSerializer
 
 
 class QuantizationWorkload(Workload):
+    """QuantizationWorkload class."""
 
     def __init__(self, data: Optional[dict] = None):
         """Initialize Quantization Workload."""
@@ -21,8 +37,10 @@ def serialize(self, serialization_type: str = "default") -> Dict[str, Any]:
 
 
 class AccuracyData(JsonSerializer):
+    """AccuracyData class."""
 
     def __init__(self, data: Optional[dict] = None):
+        """Initialize Accuracy Data."""
         super().__init__()
         if data is None:
             data = {}
@@ -31,17 +49,15 @@ def __init__(self, data: Optional[dict] = None):
 
     @property
     def baseline_accuracy(self) -> Optional[float]:
-        """
-        Get baseline accuracy.
+        """Get baseline accuracy.
 
         Returns: baseline accuracy value
         """
         return self._baseline_accuracy
 
     @property
     def optimized_accuracy(self) -> Optional[float]:
-        """
-        Get optimized accuracy.
+        """Get optimized accuracy.
 
         Returns: optimized accuracy value
         """

neural_insights/components/workload_manager/workload.py

@@ -1,3 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The workload module for Neural Insights workloads."""
+
 import os
 from datetime import datetime
 from typing import Optional, Dict, Any

neural_insights/components/workload_manager/workload_manager.py

@@ -1,3 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The workload_manager module contains code used for Neural Insights workloads management."""
+
 import json
 import os
 from os import PathLike
@@ -17,6 +33,7 @@ class WorkloadManager(JsonSerializer, metaclass=Singleton):
     """Workload Manager class."""
 
     def __init__(self, workdir_location: Optional[PathLike] = None):
+        """Initialize WorkloadManager."""
         super().__init__()
         if workdir_location is None:
             log.debug("Using default workdir location.")

neural_insights/utils/json_serializer.py

@@ -42,8 +42,7 @@ def serialize(
         self,
         serialization_type: str = "default",
     ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
-        """
-        Serialize class to dict.
+        """Serialize class to dict.
 
         :param serialization_type: serialization type, defaults to "default"
         :type serialization_type: str, optional
@@ -121,8 +120,7 @@ def _serialize_dict(
 
     @staticmethod
     def serialize_item(value: Any, serialization_type: str = "default") -> Any:
-        """
-        Serialize objects that don't support json dump.
+        """Serialize objects that don't support json dump.
 
         i.e datetime object can't be serialized to JSON format and throw an TypeError exception
         TypeError: datetime.datetime(2016, 4, 8, 11, 22, 3, 84913) is not JSON serializable

neural_insights/utils/utils.py

@@ -77,8 +77,7 @@ def get_file_extension(path: str) -> str:
 
 
 def get_framework_from_path(model_path: str) -> Optional[str]:
-    """
-    Get framework name from model extension.
+    """Get framework name from model extension.
 
     :param model_path: Path to model.
     """

neural_insights/web/configuration.py

@@ -138,8 +138,7 @@ def get_command_line_args(self) -> Dict:
         return vars(parser.parse_args())
 
     def determine_server_port(self, args: Dict) -> int:
-        """
-        Return port to be used by the server.
+        """Return port to be used by the server.
 
         Will raise a NotFoundException if port is already in use.
 
@@ -172,8 +171,7 @@ def determine_server_port(self, args: Dict) -> int:
         )
 
     def determine_gui_port(self, args: Dict) -> int:
-        """
-        Return port to be used by the GUI client.
+        """Return port to be used by the GUI client.
 
         Will return self.server_port unless specified in configuration.
         """

neural_insights/web/router.py

@@ -149,6 +149,7 @@ def get_workloads_list(data: Dict[str, Any]) -> dict:
 
 
 def get_diagnosis(workload_id: str) -> Diagnosis:
+    """Get diagnosis object for specified workload."""
     workload = WorkloadManager().get_workload(workload_id)
     diagnosis = DiagnosisFactory.get_diagnosis(workload)
     return diagnosis