diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 7e15e5741d5..a0b0faa7022 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -196,7 +196,7 @@ for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do ( if %errorlevel% neq 0 goto ERROR echo Building MSI... -msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj +"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe" /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj start %OUTPUT_DIR% diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index 67c9f6c6d8b..f470e5720ba 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -10,7 +10,9 @@ set -ex : "${CLI_VERSION:?CLI_VERSION environment variable not set.}" -: "${OUTPUT_DIR:?OUTPUT_DIR environment variable not set.}" +: "${CLI_VERSION_REVISION:?CLI_VERSION_REVISION environment variable not set.}" + +ls -Rl /mnt/artifacts WORKDIR=`cd $(dirname $0); cd ../../../; pwd` PYTHON_VERSION="3.6.5" @@ -43,7 +45,7 @@ if [ -d $WORKDIR/privates ]; then fi # The product whl are expected to be pre-built -find $OUTPUT_DIR/pypi -name '*.whl' | xargs pip3 install +find /mnt/artifacts/pypi -name '*.whl' | xargs pip3 install pip3 install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg @@ -55,4 +57,4 @@ cd $WORKDIR dpkg-buildpackage -us -uc deb_file=$WORKDIR/../azure-cli_${CLI_VERSION}-${CLI_VERSION_REVISION:=1}_all.deb -cp $deb_file ${OUTPUT_DIR} +cp $deb_file /mnt/output/ diff --git a/scripts/release/debian/pipeline.sh b/scripts/release/debian/pipeline.sh new file mode 100755 index 00000000000..05868bedfa5 --- /dev/null +++ b/scripts/release/debian/pipeline.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Build APT package in an Azure Container Instances +# This script assumes the Azure CLI is installed and logged in. + +set -ex + +: ${DISTRO:?"DISTRO is not set"} +: ${DISTRO_BASE_IMAGE:?"DISTRO_BASE_IMAGE is not set"} + +CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + +docker run --rm \ + -v "$BUILD_SOURCESDIRECTORY":/mnt/repo \ + -v "$BUILD_STAGINGDIRECTORY":/mnt/output \ + -v "$SYSTEM_ARTIFACTSDIRECTORY":/mnt/artifacts \ + -e CLI_VERSION=$CLI_VERSION \ + -e CLI_VERSION_REVISION=1~$DISTRO \ + $DISTRO_BASE_IMAGE \ + /mnt/repo/scripts/release/debian/build.sh + +if [ -d $BUILD_STAGINGDIRECTORY/pypi ]; then + rm -rf $BUILD_STAGINGDIRECTORY/pypi +fi \ No newline at end of file diff --git a/scripts/release/docker/pipeline.sh b/scripts/release/docker/pipeline.sh new file mode 100755 index 00000000000..2f5a5cb4630 --- /dev/null +++ b/scripts/release/docker/pipeline.sh @@ -0,0 +1,10 @@ +IMAGE_NAME=clibuild$BUILD_BUILDNUMBER +CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + +docker build --no-cache \ + --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" \ + --build-arg CLI_VERSION=$CLI_VERSION \ + --tag $IMAGE_NAME:latest \ + $BUILD_SOURCESDIRECTORY + +docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-${CLI_VERSION}.tar" $IMAGE_NAME:latest diff --git a/scripts/release/get_version.sh b/scripts/release/get_version.sh new file mode 100755 index 00000000000..6b9d65b11b0 --- /dev/null +++ b/scripts/release/get_version.sh @@ -0,0 +1,7 @@ +# Extract the version of the CLI from azure-cli package's __init__.py file. + +: "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}" + +ver=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` +echo $ver > $BUILD_STAGINGDIRECTORY/version +echo $ver > $BUILD_STAGINGDIRECTORY/azure-cli-${ver}.txt \ No newline at end of file diff --git a/scripts/release/homebrew/docker/formula_generate.py b/scripts/release/homebrew/docker/formula_generate.py new file mode 100644 index 00000000000..8e19de38d5f --- /dev/null +++ b/scripts/release/homebrew/docker/formula_generate.py @@ -0,0 +1,88 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import sys +import re +from typing import List, Tuple + +import requests +import jinja2 +from poet.poet import make_graph, RESOURCE_TEMPLATE + +TEMPLATE_FILE_NAME='formula_template.txt' +HOMEBREW_UPSTREAM_URL=os.environ['HOMEBREW_UPSTREAM_URL'] +HOMEBREW_FORMULAR_LATEST="https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/azure-cli.rb" + + +def main(): + print('Generate formular for Azure CLI homebrew release.') + + template_path = os.path.join(os.path.dirname(__file__), TEMPLATE_FILE_NAME) + with open(template_path, mode='r') as fq: + template_content = fq.read() + + template = jinja2.Template(template_content) + content = template.render( + upstream_url=HOMEBREW_UPSTREAM_URL, + upstream_sha=compute_sha256(HOMEBREW_UPSTREAM_URL), + resources=collect_resources(), + bottle_hash=last_bottle_hash() + ) + if not content.endswith('\n'): + content += '\n' + + with open('azure-cli.rb', mode='w') as fq: + fq.write(content) + +def compute_sha256(resource_url: str) -> str: + import hashlib + sha256 = hashlib.sha256() + resp = requests.get(resource_url) + resp.raise_for_status() + sha256.update(resp.content) + + return sha256.hexdigest() + + +def collect_resources() -> str: + nodes = make_graph('azure-cli') + nodes_render = [] + for node_name in sorted(nodes): + if not resource_filter(node_name): + continue + + nodes_render.append(RESOURCE_TEMPLATE.render(resource=nodes[node_name])) + return '\n\n'.join(nodes_render) + + +def resource_filter(name: str) -> bool: + return not name.startswith('azure-cli') and name not in ('futures') + + +def last_bottle_hash(): + """Fetch the bottle do ... end from the latest brew formula""" + resp = requests.get(HOMEBREW_FORMULAR_LATEST) + resp.raise_for_status() + + lines = resp.text.split('\n') + look_for_end = False + start = 0 + end = 0 + for idx, content in enumerate(lines): + if look_for_end: + if 'end' in content: + end = idx + break + else: + if 'bottle do' in content: + start = idx + look_for_end = True + + return '\n'.join(lines[start: end + 1]) + + +if __name__ == '__main__': + main() diff --git a/scripts/release/homebrew/docker/formula_template.txt b/scripts/release/homebrew/docker/formula_template.txt new file mode 100644 index 00000000000..2ba8606daf8 --- /dev/null +++ b/scripts/release/homebrew/docker/formula_template.txt @@ -0,0 +1,77 @@ +class AzureCli < Formula + desc "Microsoft Azure CLI 2.0" + homepage "https://docs.microsoft.com/cli/azure/overview" + url "{{ upstream_url }}" + sha256 "{{ upstream_sha }}" + head "https://github.com/Azure/azure-cli.git" + +{{ bottle_hash }} + + depends_on "openssl" + depends_on "python" + +{{ resources }} + + def install + xy = Language::Python.major_minor_version "python3" + site_packages = libexec/"lib/python#{xy}/site-packages" + ENV.prepend_create_path "PYTHONPATH", site_packages + ENV.prepend "LDFLAGS", "-L#{Formula["openssl"].opt_lib}" + ENV.prepend "CFLAGS", "-I#{Formula["openssl"].opt_include}" + ENV.prepend "CPPFLAGS", "-I#{Formula["openssl"].opt_include}" + + # Get the CLI components we'll install + components = [ + buildpath/"src/azure-cli", + buildpath/"src/azure-cli-telemetry", + buildpath/"src/azure-cli-core", + buildpath/"src/azure-cli-nspkg", + buildpath/"src/azure-cli-command_modules-nspkg", + ] + components += Pathname.glob(buildpath/"src/command_modules/azure-cli-*/") + + # Install dependencies + # note: Even if in 'resources', don't include 'futures' as not needed for Python3 + # and causes import errors. See https://github.com/agronholm/pythonfutures/issues/41 + deps = resources.map(&:name).to_set - ["futures"] + deps.each do |r| + resource(r).stage do + system "python3", *Language::Python.setup_install_args(libexec) + end + end + + # Install CLI + components.each do |item| + cd item do + system "python3", *Language::Python.setup_install_args(libexec) + end + end + + # This replaces the `import pkg_resources` namespace imports from upstream + # with empty string as the import is slow and not needed in this environment. + File.open(site_packages/"azure/__init__.py", "w") {} + File.open(site_packages/"azure/cli/__init__.py", "w") {} + File.open(site_packages/"azure/cli/command_modules/__init__.py", "w") {} + File.open(site_packages/"azure/mgmt/__init__.py", "w") {} + + (bin/"az").write <<~EOS + #!/usr/bin/env bash + export PYTHONPATH="#{ENV["PYTHONPATH"]}" + if command -v python#{xy} >/dev/null 2>&1; then + python#{xy} -m azure.cli \"$@\" + else + python3 -m azure.cli \"$@\" + fi + EOS + + bash_completion.install "az.completion" => "az" + end + + test do + json_text = shell_output("#{bin}/az cloud show --name AzureCloud") + azure_cloud = JSON.parse(json_text) + assert_equal azure_cloud["name"], "AzureCloud" + assert_equal azure_cloud["endpoints"]["management"], "https://management.core.windows.net/" + assert_equal azure_cloud["endpoints"]["resourceManager"], "https://management.azure.com/" + end +end diff --git a/scripts/release/homebrew/docker/requirements.txt b/scripts/release/homebrew/docker/requirements.txt new file mode 100644 index 00000000000..8c1c0a2a71b --- /dev/null +++ b/scripts/release/homebrew/docker/requirements.txt @@ -0,0 +1,3 @@ +homebrew-pypi-poet~=0.10.0 +jinja2~=2.10 +requests~=2.18.4 diff --git a/scripts/release/homebrew/docker/run.sh b/scripts/release/homebrew/docker/run.sh new file mode 100755 index 00000000000..679abbc07db --- /dev/null +++ b/scripts/release/homebrew/docker/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +root=$(cd $(dirname $0); pwd) + +pip install wheel==0.30.0 +pip install -U pip +pip install -r $root/requirements.txt +pip install azure-cli==$CLI_VERSION -f /mnt/pypi + +pip install msrestazure==0.4.34 + +pip list + +python $root/formula_generate.py diff --git a/scripts/release/homebrew/pipeline.sh b/scripts/release/homebrew/pipeline.sh new file mode 100644 index 00000000000..1e9aedc9c2b --- /dev/null +++ b/scripts/release/homebrew/pipeline.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +root=$(cd $(dirname $0); pwd) + +set -ex + +CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` +HOMEBREW_UPSTREAM_URL=`cat $BUILD_STAGINGDIRECTORY/github/upstream_url` +TAR_NAME=azure-cli-$CLI_VERSION.tar.gz + +docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) +pypi_files=$(cd $SYSTEM_ARTIFACTSDIRECTORY/pypi; pwd) + +echo "Generating formula in docker container ... " +docker run -v $docker_files:/mnt/scripts \ + -v $pypi_files:/mnt/pypi \ + -e CLI_VERSION=$CLI_VERSION \ + -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ + --name azurecli \ + python:3.6 \ + /mnt/scripts/run.sh + +# clean up +rm -rf $BUILD_STAGINGDIRECTORY/metadata +rm -rf $BUILD_STAGINGDIRECTORY/pypi + +docker cp azurecli:azure-cli.rb $BUILD_STAGINGDIRECTORY/azure-cli.rb +docker rm --force azurecli diff --git a/scripts/release/homebrew/upload.sh b/scripts/release/homebrew/upload.sh new file mode 100644 index 00000000000..d197f55fe0c --- /dev/null +++ b/scripts/release/homebrew/upload.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Download the release package generated by the Github when a tag of the release is created to storage account + +: {AZURE_STORAGE_ACCOUNT:?"AZURE_STORAGE_ACCOUNT not set"} +: {AZURE_STORAGE_KEY:?"AZURE_STORAGE_KEY not set"} + +CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` +TAR_NAME=azure-cli-$CLI_VERSION.tar.gz + +mkdir -p $BUILD_STAGINGDIRECTORY/github/ >/dev/null 2>&1 +curl -sL https://github.com/Azure/azure-cli/archive/$TAR_NAME -o $BUILD_STAGINGDIRECTORY/github/$TAR_NAME + +az storage blob upload -c releases -n $TAR_NAME -f $BUILD_STAGINGDIRECTORY/github/$TAR_NAME + +# Make sure the URL is publicly available +HOMEBREW_UPSTREAM_URL=$(az storage blob url -c releases -n $TAR_NAME -otsv) +curl -sfS -I $HOMEBREW_UPSTREAM_URL >/dev/null +echo -n $HOMEBREW_UPSTREAM_URL > $BUILD_STAGINGDIRECTORY/github/upstream_url diff --git a/scripts/release/pypi/build.sh b/scripts/release/pypi/build.sh index 0b6c580d721..83f8a955861 100755 --- a/scripts/release/pypi/build.sh +++ b/scripts/release/pypi/build.sh @@ -6,15 +6,20 @@ set -e -WORKDIR=`cd $(dirname $0); cd ../../../; pwd` -: ${OUTPUT_DIR:=$WORKDIR/artifacts} +: "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}" +: "${BUILD_SOURCESDIRECTORY:=`cd $(dirname $0); cd ../../../; pwd`}" -mkdir -p $OUTPUT_DIR +cd $BUILD_SOURCESDIRECTORY + +echo "Search setup files from `pwd`." +python --version + +pip install -U pip setuptools wheel +pip list -cd $WORKDIR for setup_file in $(find src -name 'setup.py' | grep -v azure-cli-testsdk); do pushd `dirname $setup_file` - python setup.py bdist_wheel -d $OUTPUT_DIR - python setup.py sdist -d $OUTPUT_DIR + python setup.py bdist_wheel -d $BUILD_STAGINGDIRECTORY + python setup.py sdist -d $BUILD_STAGINGDIRECTORY popd done diff --git a/scripts/release/rpm/build.sh b/scripts/release/rpm/build.sh index 28b39e4be47..ea6feb0e64f 100755 --- a/scripts/release/rpm/build.sh +++ b/scripts/release/rpm/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -: ${OUTPUT_DIR:=$WORKDIR/artifacts} +: "${CLI_VERSION:?CLI_VERSION environment variable not set.}" yum check-update yum install -y gcc rpm-build rpm-level rpmlint make bash corutils diffutils \ @@ -10,4 +10,4 @@ yum install -y gcc rpm-build rpm-level rpmlint make bash corutils diffutils \ set -e export REPO_PATH=`cd $(dirname $0); cd ../../../; pwd` -rpmbuild -v -bb --clean $REPO_PATH/scripts/release/rpm/azure-cli.spec && cp /root/rpmbuild/RPMS/x86_64/* $OUTPUT_DIR \ No newline at end of file +rpmbuild -v -bb --clean $REPO_PATH/scripts/release/rpm/azure-cli.spec && cp /root/rpmbuild/RPMS/x86_64/* /mnt/output \ No newline at end of file diff --git a/scripts/release/rpm/pipeline.sh b/scripts/release/rpm/pipeline.sh new file mode 100755 index 00000000000..388e93bf358 --- /dev/null +++ b/scripts/release/rpm/pipeline.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Build APT package in an Azure Container Instances +# This script assumes the Azure CLI is installed and logged in. + +set -ex + +CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + +docker run --rm \ + -v "$BUILD_SOURCESDIRECTORY":/mnt/repo \ + -v "$BUILD_STAGINGDIRECTORY":/mnt/output \ + -e CLI_VERSION=$CLI_VERSION \ + centos:7 \ + /mnt/repo/scripts/release/rpm/build.sh