Add more tools and revamp sdk-tools package (#2870)

* Add more tools and revamp sdk-tools package

* Add pyopenssl for old tests

azure-sdk-tools/packaging_tools/drop_tools.py

@@ -0,0 +1,132 @@
+"""This file is specific to Azure SDK for Python and should be split somewhere else."""
+import logging
+from pathlib import Path
+import subprocess
+import tempfile
+
+from github import Github
+
+from azure_devtools.ci_tools.github_tools import (
+    manage_git_folder,
+    DashboardCommentableObject
+)
+
+_LOGGER = logging.getLogger(__name__)
+
+
+_STORAGE_ACCOUNT = "http://azuresdkinfrajobstore1.blob.core.windows.net/azure/azure-sdk-for-python/pullrequests/{prnumber}/dist/{file}"
+
+def execute_simple_command(cmd_line, cwd=None, shell=False, env=None):
+    try:
+        process = subprocess.Popen(cmd_line,
+                                   stderr=subprocess.STDOUT,
+                                   stdout=subprocess.PIPE,
+                                   universal_newlines=True,
+                                   cwd=cwd,
+                                   shell=shell,
+                                   env=env)
+        output_buffer = []
+        for line in process.stdout:
+            output_buffer.append(line.rstrip())
+            _LOGGER.info(output_buffer[-1])
+        process.wait()
+        output = "\n".join(output_buffer)
+        if process.returncode:
+            raise subprocess.CalledProcessError(
+                process.returncode,
+                cmd_line,
+                output
+            )
+        return output
+    except Exception as err:
+        _LOGGER.error(err)
+        raise
+    else:
+        _LOGGER.info("Return code: %s", process.returncode)
+
+def build_package_from_pr_number(gh_token, sdk_id, pr_number, output_folder, *, with_comment=False):
+    """Will clone the given PR branch and vuild the package with the given name."""
+
+    con = Github(gh_token)
+    repo = con.get_repo(sdk_id)
+    sdk_pr = repo.get_pull(pr_number)
+    # "get_files" of Github only download the first 300 files. Might not be enough.
+    package_names = {f.filename.split('/')[0] for f in sdk_pr.get_files() if f.filename.startswith("azure")}
+    absolute_output_folder = Path(output_folder).resolve()
+
+    with tempfile.TemporaryDirectory() as temp_dir, \
+            manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), sdk_id, pr_number=pr_number) as sdk_folder:
+
+        for package_name in package_names:
+            _LOGGER.debug("Build {}".format(package_name))
+            execute_simple_command(
+                ["python", "./build_package.py", "--dest", str(absolute_output_folder), package_name],
+                cwd=sdk_folder
+            )
+            _LOGGER.debug("Build finished: {}".format(package_name))
+
+    if with_comment:
+        files = [f.name for f in absolute_output_folder.iterdir()]
+        comment_message = None
+        dashboard = DashboardCommentableObject(sdk_pr, "(message created by the CI based on PR content)")
+        try:
+            installation_message = build_installation_message(sdk_pr)
+            download_message = build_download_message(sdk_pr, files)
+            comment_message = installation_message + "\n\n" + download_message
+            dashboard.create_comment(comment_message)
+        except Exception:
+            _LOGGER.critical("Unable to do PR comment:\n%s", comment_message)
+
+def build_download_message(sdk_pr, files):
+    if not files:
+        return ""
+    message = "# Direct download\n\nYour files can be directly downloaded here:\n\n"
+    for filename in files:
+        message += "- [{}]({})\n".format(
+            filename,
+            _STORAGE_ACCOUNT.format(prnumber=sdk_pr.number, file=filename)
+        )
+    return message
+
+def build_installation_message(sdk_pr):
+    # Package starts with "azure" and is at the root of the repo
+    package_names = {f.filename.split('/')[0] for f in sdk_pr.get_files() if f.filename.startswith("azure")}
+
+    result = ["# Installation instruction"]
+    for package in package_names:
+        result.append("## Package {}".format(package))
+        result.append(pr_message_for_package(sdk_pr, package))
+    return "\n".join(result)
+
+
+def pr_message_for_package(sdk_pr, package_name):
+    git_path = '"git+{}@{}#egg={}&subdirectory={}"'.format(
+        sdk_pr.head.repo.html_url,
+        sdk_pr.head.ref,
+        package_name,
+        package_name
+    )
+
+    pip_install = 'pip install {}'
+    pip_wheel = 'pip wheel --no-deps {}'
+
+    pr_body = "You can install the package `{}` of this PR using the following command:\n\t`{}`".format(
+        package_name,
+        pip_install.format(git_path)
+    )
+
+    pr_body += "\n\n"
+
+    pr_body += "You can build a wheel to distribute for test using the following command:\n\t`{}`".format(
+        pip_wheel.format(git_path)
+    )
+
+    pr_body += "\n\n"
+    pr_body += "If you have a local clone of this repository, you can also do:\n\n"
+    pr_body += "- `git checkout {}`\n".format(sdk_pr.head.ref)
+    pr_body += "- `pip install -e ./{}`\n".format(package_name)
+    pr_body += "\n\n"
+    pr_body += "Or build a wheel file to distribute for testing:\n\n"
+    pr_body += "- `git checkout {}`\n".format(sdk_pr.head.ref)
+    pr_body += "- `pip wheel --no-deps ./{}`\n".format(package_name)
+    return pr_body

azure-sdk-tools/packaging_tools/generate_package.py

@@ -0,0 +1,50 @@
+import argparse
+import logging
+import os
+
+from packaging_tools.drop_tools import build_package_from_pr_number
+
+_LOGGER = logging.getLogger(__name__)
+
+def generate_main():
+    """Main method"""
+
+    parser = argparse.ArgumentParser(
+        description='Build package.',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('--pr-number', '-p',
+                        dest='pr_number', type=int, required=True,
+                        help='PR number')
+    parser.add_argument('--repo', '-r',
+                        dest='repo_id', default="Azure/azure-sdk-for-python",
+                        help='Repo id. [default: %(default)s]')
+    parser.add_argument("--with-comment",
+                        dest="with_comment", action="store_true",
+                        help="Do a comment to the original PR with info.")
+    parser.add_argument("-v", "--verbose",
+                        dest="verbose", action="store_true",
+                        help="Verbosity in INFO mode")
+    parser.add_argument("--debug",
+                        dest="debug", action="store_true",
+                        help="Verbosity in DEBUG mode")
+
+    parser.add_argument('--output-folder', '-o',
+                        dest='output_folder', default='.',
+                        help='Output folder for package. [default: %(default)s]')
+
+    args = parser.parse_args()
+    main_logger = logging.getLogger()
+    if args.verbose or args.debug:
+        logging.basicConfig()
+        main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+
+    build_package_from_pr_number(
+        os.environ.get("GH_TOKEN", None),
+        args.repo_id,
+        args.pr_number,
+        args.output_folder,
+        with_comment=args.with_comment
+    )
+
+if __name__ == "__main__":
+    generate_main()

azure-sdk-tools/setup.py

@@ -4,12 +4,39 @@
 # This is a "fake" package, meaning it's not supposed to be released but used
 # locally with "pip install -e"
 
+DEPENDENCIES = [
+    # Packaging
+    'packaging',
+    'wheel',
+    'Jinja2',
+    'pytoml',
+    'json-delta>=2.0',
+    'azure-devtools[ci_tools]>=1.1.0',
+    # Tests
+    'pytest-cov',
+    'pytest>=3.5.1',
+    # 'azure-devtools>=0.4.1' override by packaging needs
+    'readme_renderer',
+
+    # Should not be here, but split per package once they have test dependencies
+    'azure-storage-blob', # azure-servicemanagement-legacy azure-keyvault
+    'azure-storage-file', # azure-mgmt-batchai
+    'azure-storage-common', # azure-keyvault
+    'pyopenssl' # azure-servicemanagement-legacy
+]
+
 setup(
     name = "azure-sdk-tools",
     version = "0.0.0",
     author='Microsoft Corporation',
     author_email='[email protected]',
     url='https://github.com/Azure/azure-sdk-for-python',
     packages=find_packages(),
-    long_description="Specific tools for Azure SDK for Python testing"
+    long_description="Specific tools for Azure SDK for Python testing",
+    install_requires=DEPENDENCIES,
+    entry_points = {
+        'console_scripts': [
+            'generate_package=packaging_tools.generate_package:generate_main',
+        ],
+    }
 )

azure-sdk-tools/tests/test_python_sdk_tools.py

@@ -0,0 +1,23 @@
+from subprocess import CalledProcessError
+import tempfile
+
+import pytest
+
+@pytest.mark.skip(reason="test for SDK team that should be manually activated")
+def test_build_package_from_pr_number(github_token):
+    from pathlib import Path
+    from packaging_tools.drop_tools import build_package_from_pr_number
+
+    # Should build package azure-mgmt-advisor 1.0.1
+    with tempfile.TemporaryDirectory() as temp_dir:
+        build_package_from_pr_number(github_token, "Azure/azure-sdk-for-python", 2417, temp_dir)
+        temp_dir_path = Path(temp_dir)
+        files = set(file.relative_to(temp_dir) for file in temp_dir_path.iterdir())
+        assert files == {
+            Path("azure_mgmt_iothubprovisioningservices-0.2.0-py2.py3-none-any.whl"),
+            Path("azure-mgmt-iothubprovisioningservices-0.2.0.zip")
+        }
+
+    # This PR is broken and can't be built: 2040
+    with tempfile.TemporaryDirectory() as temp_dir, pytest.raises(CalledProcessError):
+        build_package_from_pr_number(github_token, "Azure/azure-sdk-for-python", 2040, temp_dir)

scripts/dev_setup.py

@@ -41,20 +41,11 @@ def pip_command(command):
 print('Running dev setup...')
 print('Root directory \'{}\'\n'.format(root_dir))
 
-# install general requirements
-pip_command('install -r requirements.txt')
-
 # install packages
 for package_list in [nspkg_packages, content_packages]:
     for package_name in package_list:
         pip_command('install -e {}'.format(package_name))
 
-# install test requirements
-pip_command('install -r azure-sdk-tools/test-requirements.txt')
-
-# install packaging requirements
-pip_command('install -r azure-sdk-tools/packaging_requirements.txt')
-
 # Ensure that the site package's azure/__init__.py has the old style namespace
 # package declaration by installing the old namespace package
 pip_command('install --force-reinstall azure-mgmt-nspkg==1.0.0')