azure-pipelines.yml

resources:
- repo: self

trigger:
  batch: true
  branches:
    include:
      - '*'

pr:
  branches:
    include:
      - '*'

jobs:
- job: CredScan
  displayName: "Credential Scan"
  pool:
    name: 'pool-windows-2019'
  steps:
  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
    displayName: 'Run Credential Scanner'
    inputs:
      toolVersion: latest
      suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json'
      
  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
    displayName: 'Post Analysis'
    inputs:
      GdnBreakAllTools: false
      GdnBreakGdnToolCredScan: true
      GdnBreakGdnToolCredScanSeverity: Error

- job: PolicyCheck
  displayName: "Policy Check"
  pool:
    name: 'pool-windows-2019'
  steps:
  - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2
    displayName: 'Run Policy Check'
    inputs:
      targetType: F
      result: PoliCheckResult.xml
  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
    displayName: 'Post Analysis'
    inputs:
      GdnBreakAllTools: false
      GdnBreakGdnToolPoliCheck: true
      GdnBreakGdnToolPoliCheckSeverity: Error

- job: CheckLicenseHeader
  displayName: "Check License"
  pool:
    name: 'pool-ubuntu-2004'
  steps:
  - task: UsePythonVersion@0
    displayName: 'Use Python 3.12'
    inputs:
      versionSpec: 3.12
  - template: .azure-pipelines/templates/azdev_setup.yml
  - bash: |
      #!/usr/bin/env bash
      set -ev
      source ./env/bin/activate
      azdev verify license

- job: IndexVerify
  displayName: "Verify Extensions Index"
  pool:
    name: 'pool-ubuntu-2004'
  steps:
  - task: UsePythonVersion@0
    displayName: 'Use Python 3.12'
    inputs:
      versionSpec: 3.12
  - bash: |
      #!/usr/bin/env bash
      set -ev
      pip install wheel==0.30.0 requests packaging setuptools
      export CI="ADO"
      python ./scripts/ci/test_index.py -v
    displayName: "Verify Extensions Index"

- job: SourceTests
  displayName: "Integration Tests, Build Tests"
  pool:
    name: 'pool-ubuntu-2004'
  strategy:
    matrix:
      Python38:
        python.version: '3.8'
      Python39:
        python.version: '3.9'
      Python310:
        python.version: '3.10'
      Python311:
        python.version: '3.12'
  steps:
    - task: UsePythonVersion@0
      displayName: 'Use Python $(python.version)'
      inputs:
        versionSpec: '$(python.version)'
    - template: .azure-pipelines/templates/azdev_setup.yml
    - bash: pip install wheel==0.30.0
      displayName: 'Install wheel==0.30.0'
    - bash: |
        #!/usr/bin/env bash
        set -ev
        source ./env/bin/activate
        az --version
        python scripts/ci/test_source.py -v
      displayName: 'Run integration test and build test'
      env:
        ADO_PULL_REQUEST_LATEST_COMMIT: HEAD
        ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)

- job: AzdevStyleModifiedExtensions
  displayName: "azdev style on Modified Extensions"
  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
  continueOnError: true
  pool:
    name: 'pool-ubuntu-2004'
  steps:
    - task: UsePythonVersion@0
      displayName: 'Use Python 3.12'
      inputs:
        versionSpec: 3.12
    - template: .azure-pipelines/templates/azdev_setup.yml
    - bash: |
        #!/usr/bin/env bash
        set -ev
        source ./env/bin/activate
        # overwrite the default AZURE_EXTENSION_DIR set by ADO
        AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version

        AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py --type style
      displayName: "azdev style on Modified Extensions"
      env:
        ADO_PULL_REQUEST_LATEST_COMMIT: HEAD
        ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)

- job: AzdevLinterModifiedExtensions
  displayName: "azdev linter on Modified Extensions"
  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
  pool:
    name: 'pool-ubuntu-2004'
  steps:
    - task: UsePythonVersion@0
      displayName: 'Use Python 3.12'
      inputs:
        versionSpec: 3.12
    - template: .azure-pipelines/templates/azdev_setup.yml
    - bash: |
        #!/usr/bin/env bash
        set -ev
        source ./env/bin/activate
        # overwrite the default AZURE_EXTENSION_DIR set by ADO
        AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version
        
        # TODO: remove --type linter once all extensions are fixed
        AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/azdev_linter_style.py --type linter
      displayName: "azdev linter on Modified Extensions"
      env:
        ADO_PULL_REQUEST_LATEST_COMMIT: HEAD
        ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)

- job: AzdevScanModifiedExtensions
  displayName: "azdev scan on Modified Extensions"
  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
  pool:
    name: 'pool-ubuntu-2004'
  steps:
    - task: UsePythonVersion@0
      displayName: 'Use Python 3.11'
      inputs:
        versionSpec: 3.11
    - template: .azure-pipelines/templates/azdev_setup.yml
    - bash: |
        #!/usr/bin/env bash
        set -ev
        source ./env/bin/activate
        git fetch origin --depth=1 $(System.PullRequest.TargetBranch)
        declare -A secret_files
        for FILE in `git diff --name-only --diff-filter=AM origin/$(System.PullRequest.TargetBranch)` ; do
          detected=$(azdev scan -f $FILE | python -c "import sys, json; print(json.load(sys.stdin)['secrets_detected'])")
          if [ $detected == 'True' ]; then
            echo "Secrets detected from "$FILE", Please remove or replace it. You can run 'azdev scan'/'azdev mask' locally to fix." 1>&2
            secret_files+=$FILE
          fi
        done
        if [ "${#secret_files[@]}" -gt 0 ]; then
          exit 1
        fi
      displayName: "azdev scan on Modified Extensions"

#- job: IndexRefDocVerify
#  displayName: "Verify Ref Docs"
#  continueOnError: true
#  pool:
#    name: 'pool-ubuntu-2004'
#  steps:
#  - task: UsePythonVersion@0
#    displayName: 'Use Python 3.12'
#    inputs:
#      versionSpec: 3.12
#  - bash: pip install wheel==0.30.0
#    displayName: 'Install wheel==0.30.0'
#  - task: Bash@3
#    displayName: "Verify Extension Ref Docs"
#    inputs:
#      targetType: 'filePath'
#      filePath: scripts/ci/test_index_ref_doc.sh

- job: CheckInit
  displayName: "Check Init Files"
  pool:
    name: 'pool-ubuntu-2004'
  steps:
  - task: UsePythonVersion@0
    displayName: 'Use Python 3.x'
    inputs:
      versionSpec: 3.x
  - bash: |
      python scripts/ci/test_init.py -v