azure-pipelines.yml

name: $(Build.SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
trigger: [ master ]
pool:
  vmImage: 'Ubuntu 16.04'

# Variables to use in build tasks
variables:
  serviceConnectionAzureSubscription: nameOfTheServiceConnection # replace with Azure DevOps Azure ARM service connection name
  keyVaultName: nameOfKeyVault # replace with name of KeyVault used to store CI/CD pipeline secrets
  secretsFilter: 'CustodianBuildServicePrincipal' # replace with name of secret or * for all secrets
  policiesFilePath: policies/policies.yml # replace with name of yml file containing the policies that need to be validated
  configFilePath: policies/config.json # replace with path to policy config file if not default
  policyFilePath: policies/policies.build.json # replace with path to policy file if not default
  outputPath: output # replace with path for Custodian policy run output
  repositoryId: idOfTheRepository # replace with the id of the repository, see Project Settings -> Repositories -> Select repo and look at URL
  cloudCustodianPipVersion: '0.8.32.0'  # Cloud Custodian pypi version to use
  cloudCustodianAzurePipVersion: '0.3'  # Cloud Custodian Azure pypi version to use
  validatePolicyChangesOnly: True

# Build tasks
steps:
  # Set Python Version
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '3.6'

  # Fetch Key Vault Secrets
  - task: AzureKeyVault@1
    displayName: 'Download Azure Key Vault Secrets'
    inputs:
      azureSubscription: $(serviceConnectionAzureSubscription)
      keyVaultName: $(keyVaultName)
      secretsFilter: $(secretsFilter)

  # Update python tools
  - script: python -m pip install --upgrade pip setuptools wheel
    displayName: 'Update python pip tools'

  # Install libgit2 for policystream.py
  - script: sudo apt-get update && sudo apt-get -y install libgit2-dev
    displayName: 'Install libgit2'

  # Install dependencies for policy stream
  - script: pip install -r requirements.txt
    displayName: 'Install pip dependencies'

  # Install Cloud Custodian and c7n_azure
  - script: mkdir -p $AGENT_BUILDDIRECTORY/temp &&
      cd $AGENT_BUILDDIRECTORY/temp &&
      pip download --no-deps c7n==$(cloudCustodianPipVersion) &&
      pip download --no-deps c7n_azure==$(cloudCustodianAzurePipVersion) &&
      mv c7n-$(cloudCustodianPipVersion).tar.gz c7n.tar.gz &&
      mv c7n_azure-$(cloudCustodianAzurePipVersion).tar.gz c7n_azure.tar.gz &&
      pip install ./c7n.tar.gz &&
      pip install ./c7n_azure.tar.gz
    displayName: 'Install Cloud Custodian'

  # Get c7n-policystream python package
  - script: pip install c7n-policystream
    displayName: 'Get c7n-policystream python package'

  # Run c7n-policystream to get modified policies by default
  - script: |
      case $(Build.Reason) in
      "PullRequest"*)
        if [ "$(git diff origin/master --name-only | grep -c "policies/config.json")" -gt 0 ];
        then
          echo "Detected change in 'policies/config.json'. Getting all custodian policies"
          python src/build/scripts/get_all_policies.py -r . -o $(policiesFilePath)
        else
          echo "Getting modified policies for pull request"
          c7n-policystream diff -r . --source origin/master --target HEAD -o $(policiesFilePath)
        fi
        ;;
      *)
        LAST_RELEASED_COMMIT_ID=$(./src/build/scripts/getLastReleasedCommit.sh "$(System.TeamFoundationCollectionUri)" "$(System.TeamProject)" "$(System.TeamProjectId)" $(System.DefinitionId) $(System.AccessToken))

        if [ -z "$LAST_RELEASED_COMMIT_ID" ]
        then
          echo "Couldn't get the last released commit id. Getting all custodian policies"
          python src/build/scripts/get_all_policies.py -r . -o $(policiesFilePath)
        elif [ "$(git diff $LAST_RELEASED_COMMIT_ID --name-only  | grep -c "policies/config.json")" -gt 0 ];
        then
          echo "Detected change in 'policies/config.json'. Getting all custodian policies"
          python src/build/scripts/get_all_policies.py -r . -o $(policiesFilePath)
        else
          echo "Getting modified policies since the last released commit id $LAST_RELEASED_COMMIT_ID"
          c7n-policystream diff -r . --source $LAST_RELEASED_COMMIT_ID -o $(policiesFilePath)
        fi
      esac
    displayName: 'Get all custodian policies that have been modified'
    condition: and(succeeded(), eq(variables['validatePolicyChangesOnly'], True))

  # Gather all policies for validation when validatePolicyChangesOnly is false
  - task: PythonScript@0
    displayName: 'Get all custodian policies'
    inputs:
      scriptSource: 'filePath'
      scriptPath: 'src/build/scripts/get_all_policies.py'
      arguments: '-r . -o $(policiesFilePath)'
    condition: and(succeeded(), eq(variables['validatePolicyChangesOnly'], False))

  # Run Custodian validate
  - script: 'custodian validate $(policiesFilePath)'
    displayName: 'Validate policies'

  # Check if all policies are Function modes
  - task: PythonScript@0
    displayName: 'Validate that policies are in function mode'
    inputs:
      scriptSource: 'filePath'
      scriptPath: "src/build/scripts/validate_policy_mode.py"
      arguments: '-m $(policiesFilePath)'

  # Run Custodian policies in Dryrun mode
  - task: PythonScript@0
    displayName: 'Dry run Custodian policies to get all resources that will be affected'
    inputs:
      scriptSource: 'filePath'
      scriptPath: "src/build/scripts/policy_runner.py"
      arguments: '-c $(configFilePath) -p $(policyFilePath) -s $(CustodianBuildServicePrincipal) -f $(CustodianBuildServicePrincipal) -o $(outputPath) --dryrun'
      failOnStderr: true

  # Post the output of the dry run to the PR
  - task: PythonScript@0
    displayName: 'Post output to PR'
    inputs:
      scriptSource: 'filePath'
      scriptPath: 'src/build/scripts/post_to_pr.py'
      arguments: '-o "$(System.TeamFoundationCollectionUri)" -p "$(System.TeamProject)" -r "$(repositoryId)" -i "$(System.PullRequest.PullRequestId)" -t "$(System.AccessToken)" -d "$(outputPath)" --enable-vso-output'

  # Copy c7n and c7n_azure python package artifacts
  - script: mkdir -p $BUILD_ARTIFACTSTAGINGDIRECTORY/c7n-dist &&
      cp $AGENT_BUILDDIRECTORY/temp/c7n*.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY/c7n-dist/
    displayName: 'Copy c7n and c7n_azure python package artifacts'

  # Copy Cloud Custodian dryrun output files
  - task: CopyFiles@2
    displayName: 'Copy Cloud Custodian dryrun output files'
    inputs:
      contents: 'output/**/*'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  # Copy configuration files
  - task: CopyFiles@2
    displayName: 'Copy configuration files'
    inputs:
      contents: 'policies/*.json'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  # Copy deployment scripts
  - task: CopyFiles@2
    displayName: 'Copy deployment scripts'
    inputs:
      contents: 'src/build/scripts/policy_runner.py'
      targetFolder: '$(Build.ArtifactStagingDirectory)'
      flattenFolders: true

  # Copy policies to deploy
  - task: CopyFiles@2
    displayName: 'Copy policies to deploy'
    inputs:
      contents: '$(policiesFilePath)'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  # Publish Build Artifacts
  - task: PublishBuildArtifacts@1
    inputs:
      PathToPublish: '$(Build.ArtifactStagingDirectory)'
      ArtifactName: package
      publishLocation: Container