[Infra] Add scripts supporting Azure Pipeline (#7471)

build_scripts/windows/scripts/build.cmd

@@ -196,7 +196,7 @@ for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do (
 if %errorlevel% neq 0 goto ERROR
 
 echo Building MSI...
-msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj
+"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe" /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj
 
 start %OUTPUT_DIR%
 

scripts/release/debian/build.sh

@@ -10,7 +10,9 @@
 set -ex
 
 : "${CLI_VERSION:?CLI_VERSION environment variable not set.}"
-: "${OUTPUT_DIR:?OUTPUT_DIR environment variable not set.}"
+: "${CLI_VERSION_REVISION:?CLI_VERSION_REVISION environment variable not set.}"
+
+ls -Rl /mnt/artifacts
 
 WORKDIR=`cd $(dirname $0); cd ../../../; pwd`
 PYTHON_VERSION="3.6.5"
@@ -43,7 +45,7 @@ if [ -d $WORKDIR/privates ]; then
 fi
 
 # The product whl are expected to be pre-built
-find $OUTPUT_DIR/pypi -name '*.whl' | xargs pip3 install
+find /mnt/artifacts/pypi -name '*.whl' | xargs pip3 install
 
 pip3 install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
 
@@ -55,4 +57,4 @@ cd $WORKDIR
 dpkg-buildpackage -us -uc
 
 deb_file=$WORKDIR/../azure-cli_${CLI_VERSION}-${CLI_VERSION_REVISION:=1}_all.deb
-cp $deb_file ${OUTPUT_DIR}
+cp $deb_file /mnt/output/

scripts/release/debian/pipeline.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Build APT package in an Azure Container Instances
+# This script assumes the Azure CLI is installed and logged in.
+
+set -ex
+
+: ${DISTRO:?"DISTRO is not set"}
+: ${DISTRO_BASE_IMAGE:?"DISTRO_BASE_IMAGE is not set"}
+
+CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+
+docker run --rm \
+           -v "$BUILD_SOURCESDIRECTORY":/mnt/repo \
+           -v "$BUILD_STAGINGDIRECTORY":/mnt/output \
+           -v "$SYSTEM_ARTIFACTSDIRECTORY":/mnt/artifacts \
+           -e CLI_VERSION=$CLI_VERSION \
+           -e CLI_VERSION_REVISION=1~$DISTRO \
+           $DISTRO_BASE_IMAGE \
+           /mnt/repo/scripts/release/debian/build.sh
+
+if [ -d $BUILD_STAGINGDIRECTORY/pypi ]; then
+    rm -rf $BUILD_STAGINGDIRECTORY/pypi
+fi

scripts/release/docker/pipeline.sh

@@ -0,0 +1,10 @@
+IMAGE_NAME=clibuild$BUILD_BUILDNUMBER
+CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+
+docker build --no-cache \
+             --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" \
+             --build-arg CLI_VERSION=$CLI_VERSION \
+             --tag $IMAGE_NAME:latest \
+             $BUILD_SOURCESDIRECTORY
+
+docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-${CLI_VERSION}.tar" $IMAGE_NAME:latest

scripts/release/get_version.sh

@@ -0,0 +1,7 @@
+# Extract the version of the CLI from azure-cli package's __init__.py file.
+
+: "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}"
+
+ver=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+echo $ver > $BUILD_STAGINGDIRECTORY/version
+echo $ver > $BUILD_STAGINGDIRECTORY/azure-cli-${ver}.txt

scripts/release/homebrew/docker/formula_generate.py

@@ -0,0 +1,88 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import sys
+import re
+from typing import List, Tuple
+
+import requests
+import jinja2
+from poet.poet import make_graph, RESOURCE_TEMPLATE
+
+TEMPLATE_FILE_NAME='formula_template.txt'
+HOMEBREW_UPSTREAM_URL=os.environ['HOMEBREW_UPSTREAM_URL']
+HOMEBREW_FORMULAR_LATEST="https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/azure-cli.rb"
+
+
+def main():
+    print('Generate formular for Azure CLI homebrew release.')
+
+    template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_FILE_NAME)
+    with open(template_path, mode='r') as fq:
+        template_content = fq.read()
+
+    template = jinja2.Template(template_content)
+    content = template.render(
+        upstream_url=HOMEBREW_UPSTREAM_URL,
+        upstream_sha=compute_sha256(HOMEBREW_UPSTREAM_URL),
+        resources=collect_resources(),
+        bottle_hash=last_bottle_hash()
+    )
+    if not content.endswith('\n'):
+        content += '\n'
+
+    with open('azure-cli.rb', mode='w') as fq:
+        fq.write(content)
+
+def compute_sha256(resource_url: str) -> str:
+    import hashlib
+    sha256 = hashlib.sha256()
+    resp = requests.get(resource_url)
+    resp.raise_for_status()
+    sha256.update(resp.content)
+
+    return sha256.hexdigest()
+
+
+def collect_resources() -> str:
+    nodes = make_graph('azure-cli')
+    nodes_render = []
+    for node_name in sorted(nodes):
+        if not resource_filter(node_name):
+            continue
+
+        nodes_render.append(RESOURCE_TEMPLATE.render(resource=nodes[node_name]))
+    return '\n\n'.join(nodes_render)
+
+
+def resource_filter(name: str) -> bool:
+    return not name.startswith('azure-cli') and name not in ('futures')
+
+
+def last_bottle_hash():
+    """Fetch the bottle do ... end from the latest brew formula"""
+    resp = requests.get(HOMEBREW_FORMULAR_LATEST)
+    resp.raise_for_status()
+
+    lines = resp.text.split('\n')
+    look_for_end = False
+    start = 0
+    end = 0
+    for idx, content in enumerate(lines):
+        if look_for_end:
+            if 'end' in content:
+                end = idx
+                break
+        else:
+            if 'bottle do' in content:
+                start = idx
+                look_for_end = True
+
+    return '\n'.join(lines[start: end + 1])
+
+
+if __name__ == '__main__':
+    main()

scripts/release/homebrew/docker/formula_template.txt

@@ -0,0 +1,77 @@
+class AzureCli < Formula
+  desc "Microsoft Azure CLI 2.0"
+  homepage "https://docs.microsoft.com/cli/azure/overview"
+  url "{{ upstream_url }}"
+  sha256 "{{ upstream_sha }}"
+  head "https://github.com/Azure/azure-cli.git"
+
+{{ bottle_hash }}
+
+  depends_on "openssl"
+  depends_on "python"
+
+{{ resources }}
+
+  def install
+    xy = Language::Python.major_minor_version "python3"
+    site_packages = libexec/"lib/python#{xy}/site-packages"
+    ENV.prepend_create_path "PYTHONPATH", site_packages
+    ENV.prepend "LDFLAGS", "-L#{Formula["openssl"].opt_lib}"
+    ENV.prepend "CFLAGS", "-I#{Formula["openssl"].opt_include}"
+    ENV.prepend "CPPFLAGS", "-I#{Formula["openssl"].opt_include}"
+
+    # Get the CLI components we'll install
+    components = [
+      buildpath/"src/azure-cli",
+      buildpath/"src/azure-cli-telemetry",
+      buildpath/"src/azure-cli-core",
+      buildpath/"src/azure-cli-nspkg",
+      buildpath/"src/azure-cli-command_modules-nspkg",
+    ]
+    components += Pathname.glob(buildpath/"src/command_modules/azure-cli-*/")
+
+    # Install dependencies
+    # note: Even if in 'resources', don't include 'futures' as not needed for Python3
+    # and causes import errors. See https://github.com/agronholm/pythonfutures/issues/41
+    deps = resources.map(&:name).to_set - ["futures"]
+    deps.each do |r|
+      resource(r).stage do
+        system "python3", *Language::Python.setup_install_args(libexec)
+      end
+    end
+
+    # Install CLI
+    components.each do |item|
+      cd item do
+        system "python3", *Language::Python.setup_install_args(libexec)
+      end
+    end
+
+    # This replaces the `import pkg_resources` namespace imports from upstream
+    # with empty string as the import is slow and not needed in this environment.
+    File.open(site_packages/"azure/__init__.py", "w") {}
+    File.open(site_packages/"azure/cli/__init__.py", "w") {}
+    File.open(site_packages/"azure/cli/command_modules/__init__.py", "w") {}
+    File.open(site_packages/"azure/mgmt/__init__.py", "w") {}
+
+    (bin/"az").write <<~EOS
+      #!/usr/bin/env bash
+      export PYTHONPATH="#{ENV["PYTHONPATH"]}"
+      if command -v python#{xy} >/dev/null 2>&1; then
+        python#{xy} -m azure.cli \"$@\"
+      else
+        python3 -m azure.cli \"$@\"
+      fi
+    EOS
+
+    bash_completion.install "az.completion" => "az"
+  end
+
+  test do
+    json_text = shell_output("#{bin}/az cloud show --name AzureCloud")
+    azure_cloud = JSON.parse(json_text)
+    assert_equal azure_cloud["name"], "AzureCloud"
+    assert_equal azure_cloud["endpoints"]["management"], "https://management.core.windows.net/"
+    assert_equal azure_cloud["endpoints"]["resourceManager"], "https://management.azure.com/"
+  end
+end

scripts/release/homebrew/docker/requirements.txt

@@ -0,0 +1,3 @@
+homebrew-pypi-poet~=0.10.0
+jinja2~=2.10
+requests~=2.18.4

scripts/release/homebrew/docker/run.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+root=$(cd $(dirname $0); pwd)
+
+pip install wheel==0.30.0
+pip install -U pip
+pip install -r $root/requirements.txt
+pip install azure-cli==$CLI_VERSION -f /mnt/pypi
+
+pip install msrestazure==0.4.34
+
+pip list
+
+python $root/formula_generate.py

scripts/release/homebrew/pipeline.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+root=$(cd $(dirname $0); pwd)
+
+set -ex
+
+CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version`
+HOMEBREW_UPSTREAM_URL=`cat $BUILD_STAGINGDIRECTORY/github/upstream_url`
+TAR_NAME=azure-cli-$CLI_VERSION.tar.gz
+
+docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd)
+pypi_files=$(cd $SYSTEM_ARTIFACTSDIRECTORY/pypi; pwd)
+
+echo "Generating formula in docker container ... "
+docker run -v $docker_files:/mnt/scripts \
+           -v $pypi_files:/mnt/pypi \
+           -e CLI_VERSION=$CLI_VERSION \
+           -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \
+           --name azurecli \
+           python:3.6 \
+           /mnt/scripts/run.sh
+
+# clean up
+rm -rf $BUILD_STAGINGDIRECTORY/metadata
+rm -rf $BUILD_STAGINGDIRECTORY/pypi
+
+docker cp azurecli:azure-cli.rb $BUILD_STAGINGDIRECTORY/azure-cli.rb
+docker rm --force azurecli

scripts/release/homebrew/upload.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# Download the release package generated by the Github when a tag of the release is created to storage account
+
+: {AZURE_STORAGE_ACCOUNT:?"AZURE_STORAGE_ACCOUNT not set"}
+: {AZURE_STORAGE_KEY:?"AZURE_STORAGE_KEY not set"}
+
+CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version`
+TAR_NAME=azure-cli-$CLI_VERSION.tar.gz
+
+mkdir -p $BUILD_STAGINGDIRECTORY/github/ >/dev/null 2>&1
+curl -sL https://github.com/Azure/azure-cli/archive/$TAR_NAME -o $BUILD_STAGINGDIRECTORY/github/$TAR_NAME
+
+az storage blob upload -c releases -n $TAR_NAME -f $BUILD_STAGINGDIRECTORY/github/$TAR_NAME
+
+# Make sure the URL is publicly available
+HOMEBREW_UPSTREAM_URL=$(az storage blob url -c releases -n $TAR_NAME -otsv)
+curl -sfS -I $HOMEBREW_UPSTREAM_URL >/dev/null
+echo -n $HOMEBREW_UPSTREAM_URL > $BUILD_STAGINGDIRECTORY/github/upstream_url

scripts/release/pypi/build.sh

@@ -6,15 +6,20 @@
 
 set -e
 
-WORKDIR=`cd $(dirname $0); cd ../../../; pwd`
-: ${OUTPUT_DIR:=$WORKDIR/artifacts}
+: "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}"
+: "${BUILD_SOURCESDIRECTORY:=`cd $(dirname $0); cd ../../../; pwd`}"
 
-mkdir -p $OUTPUT_DIR
+cd $BUILD_SOURCESDIRECTORY
+
+echo "Search setup files from `pwd`."
+python --version
+
+pip install -U pip setuptools wheel
+pip list
 
-cd $WORKDIR
 for setup_file in $(find src -name 'setup.py' | grep -v azure-cli-testsdk); do
     pushd `dirname $setup_file`
-    python setup.py bdist_wheel -d $OUTPUT_DIR
-    python setup.py sdist -d $OUTPUT_DIR
+    python setup.py bdist_wheel -d $BUILD_STAGINGDIRECTORY
+    python setup.py sdist -d $BUILD_STAGINGDIRECTORY
     popd
 done

scripts/release/rpm/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-: ${OUTPUT_DIR:=$WORKDIR/artifacts}
+: "${CLI_VERSION:?CLI_VERSION environment variable not set.}"
 
 yum check-update
 yum install -y gcc rpm-build rpm-level rpmlint make bash corutils diffutils \
@@ -10,4 +10,4 @@ yum install -y gcc rpm-build rpm-level rpmlint make bash corutils diffutils \
 set -e
 
 export REPO_PATH=`cd $(dirname $0); cd ../../../; pwd`
-rpmbuild -v -bb --clean $REPO_PATH/scripts/release/rpm/azure-cli.spec && cp /root/rpmbuild/RPMS/x86_64/* $OUTPUT_DIR
+rpmbuild -v -bb --clean $REPO_PATH/scripts/release/rpm/azure-cli.spec && cp /root/rpmbuild/RPMS/x86_64/* /mnt/output

scripts/release/rpm/pipeline.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Build APT package in an Azure Container Instances
+# This script assumes the Azure CLI is installed and logged in.
+
+set -ex
+
+CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+
+docker run --rm \
+           -v "$BUILD_SOURCESDIRECTORY":/mnt/repo \
+           -v "$BUILD_STAGINGDIRECTORY":/mnt/output \
+           -e CLI_VERSION=$CLI_VERSION \
+           centos:7 \
+           /mnt/repo/scripts/release/rpm/build.sh