From 2bdbbbe891e2ef3fee966dcb1c3ea97ca1805a3d Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 11:48:24 -0700 Subject: [PATCH 01/14] Enable APIStubGen step --- eng/pipelines/templates/steps/analyze.yml | 10 +++++----- eng/tox/tox.ini | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 55461845c5fe..729110e04c42 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -91,8 +91,8 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} - # - template: ../steps/run_apistub.yml - # parameters: - # ServiceDirectory: ${{ parameters.ServiceDirectory }} - # BuildTargetingString: ${{ parameters.BuildTargetingString }} - # TestMarkArgument: ${{ parameters.TestMarkArgument }} + - template: ../steps/run_apistub.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index f9ab076ca8cc..54b780681ba1 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -228,7 +228,7 @@ commands = skipsdist = false skip_install = false usedevelop = true -changedir = {toxinidir} +changedir = {envtmpdir} deps = {[base]deps} commands = From d527adecdd458ef07a28998f18b6ff5940f77cb0 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 16:46:47 -0700 Subject: [PATCH 02/14] Add pip freeze --- coverage.xml | 2914 +++++++++++++++++ eng/tox/tox.ini | 3 +- .../azure-storage-nspkg/dev_requirements.txt | 1 + 3 files changed, 2917 insertions(+), 1 deletion(-) create mode 100644 coverage.xml create mode 100644 sdk/storage/azure-storage-nspkg/dev_requirements.txt diff --git a/coverage.xml b/coverage.xml new file mode 100644 index 000000000000..3fe69bfd0dbc --- /dev/null +++ b/coverage.xml @@ -0,0 +1,2914 @@ + + + + + + C:\git\azure-sdk-for-python + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 54b780681ba1..314c2b6d282f 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -228,10 +228,11 @@ commands = skipsdist = false skip_install = false usedevelop = true -changedir = {envtmpdir} +changedir = {toxworkdir} deps = {[base]deps} commands = # install API stub generator {envbindir}/python -m pip install "git+https://github.com/Azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" + {envbindir}/python -m pip freeze apistubgen --pkg-path {toxinidir} --temp-path {envtmpdir} --out-path {envtmpdir} diff --git a/sdk/storage/azure-storage-nspkg/dev_requirements.txt b/sdk/storage/azure-storage-nspkg/dev_requirements.txt new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/sdk/storage/azure-storage-nspkg/dev_requirements.txt @@ -0,0 +1 @@ + From d4c3a75b0d022a3118dda45416b6264f7a410c12 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 16:55:54 -0700 Subject: [PATCH 03/14] Revert accidental commit --- coverage.xml | 2914 ----------------- .../azure-storage-nspkg/dev_requirements.txt | 1 - 2 files changed, 2915 deletions(-) delete mode 100644 coverage.xml delete mode 100644 sdk/storage/azure-storage-nspkg/dev_requirements.txt diff --git a/coverage.xml b/coverage.xml deleted file mode 100644 index 3fe69bfd0dbc..000000000000 --- a/coverage.xml +++ /dev/null @@ -1,2914 +0,0 @@ - - - - - - C:\git\azure-sdk-for-python - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sdk/storage/azure-storage-nspkg/dev_requirements.txt b/sdk/storage/azure-storage-nspkg/dev_requirements.txt deleted file mode 100644 index 8b137891791f..000000000000 --- a/sdk/storage/azure-storage-nspkg/dev_requirements.txt +++ /dev/null @@ -1 +0,0 @@ - From fa1b7b7810cd2dfa7b5cecca338f51eedf293e00 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 18:32:08 -0700 Subject: [PATCH 04/14] Run apistubgen from temp dir as cwd --- eng/tox/run_apistubgen.py | 49 +++++++++++++++++++++++++++++++++++++++ eng/tox/tox.ini | 4 ++-- 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 eng/tox/run_apistubgen.py diff --git a/eng/tox/run_apistubgen.py b/eng/tox/run_apistubgen.py new file mode 100644 index 000000000000..7337464c53ea --- /dev/null +++ b/eng/tox/run_apistubgen.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. + +from subprocess import check_call, CalledProcessError +import argparse +import os +import logging +import sys + + +logging.getLogger().setLevel(logging.INFO) + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run apistubgen against target folder. ") + + parser.add_argument( + "-t", + "--target", + dest="target_package", + help="The target package directory on disk.", + required=True, + ) + + parser.add_argument( + "-w", + "--work_dir", + dest="work_dir", + help="Working directory to run apistubgen", + required=True, + ) + + + args = parser.parse_args() + check_call( + [ + "apistubgen", + "--pkg-path", + args.target_package, + ], + + cwd=args.work_dir + ) + + diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 314c2b6d282f..91a0c2b7d33e 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -228,11 +228,11 @@ commands = skipsdist = false skip_install = false usedevelop = true -changedir = {toxworkdir} +changedir = {envtmpdir} deps = {[base]deps} commands = # install API stub generator {envbindir}/python -m pip install "git+https://github.com/Azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" {envbindir}/python -m pip freeze - apistubgen --pkg-path {toxinidir} --temp-path {envtmpdir} --out-path {envtmpdir} + {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} From fea2a166a17f6421a3be6ea335b8265315e38a48 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 22:50:59 -0700 Subject: [PATCH 05/14] Dummy changes to enable logging --- eng/tox/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 91a0c2b7d33e..e0b27a5ead58 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -235,4 +235,4 @@ commands = # install API stub generator {envbindir}/python -m pip install "git+https://github.com/Azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" {envbindir}/python -m pip freeze - {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} + apistubgen --pkg-path {toxinidir} --temp-path {envtmpdir} --out-path {envtmpdir} --verbose From 8a31d1acba14c275ab2bd9cf5b19a55ed03745ee Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Mon, 20 Jul 2020 23:02:07 -0700 Subject: [PATCH 06/14] More debugging to drop --- eng/tox/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index e0b27a5ead58..994cb293dbef 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -233,6 +233,6 @@ deps = {[base]deps} commands = # install API stub generator - {envbindir}/python -m pip install "git+https://github.com/Azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" + {envbindir}/python -m pip install "git+https://github.com/praveenkuttappan/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" {envbindir}/python -m pip freeze apistubgen --pkg-path {toxinidir} --temp-path {envtmpdir} --out-path {envtmpdir} --verbose From dba6e1d8b31d1004b17abdb843bf387b315f9c80 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 01:02:53 -0700 Subject: [PATCH 07/14] Install package from prebuilt wheel --- .../templates/jobs/archetype-sdk-client.yml | 1 + eng/pipelines/templates/steps/analyze.yml | 2 ++ eng/pipelines/templates/steps/run_apistub.yml | 4 ++- eng/tox/run_apistubgen.py | 25 ++++++++++++++++--- eng/tox/tox.ini | 4 +-- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 3435fac571ac..8124bca8ac2f 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -73,6 +73,7 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} + AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - job: 'Test' condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true')) diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 729110e04c42..57d7b01a3d84 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -2,6 +2,7 @@ parameters: BuildTargetingString: 'azure-*' ServiceDirectory: '' TestMarkArgument: '' + AdditionalTestArgs: '' steps: - task: UsePythonVersion@0 @@ -96,3 +97,4 @@ steps: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} + AdditionalTestArgs: ${{parameters.AdditionalTestArgs}} diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index d66f7cafb551..a8691c39d809 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -2,6 +2,7 @@ parameters: BuildTargetingString: 'azure-*' ServiceDirectory: '' TestMarkArgument: '' + AdditionalTestArgs: '' steps: - task: UsePythonVersion@0 @@ -21,7 +22,8 @@ steps: inputs: scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' arguments: >- - "${{ parameters.BuildTargetingString }}" + "${{ parameters.BuildTargetingString }}" + ${{ parameters.AdditionalTestArgs }} --mark_arg="${{ parameters.TestMarkArgument }}" --service="${{ parameters.ServiceDirectory }}" --toxenv="apistub" diff --git a/eng/tox/run_apistubgen.py b/eng/tox/run_apistubgen.py index 7337464c53ea..c279f42c61c9 100644 --- a/eng/tox/run_apistubgen.py +++ b/eng/tox/run_apistubgen.py @@ -10,11 +10,24 @@ import logging import sys +from tox_helper_tasks import find_whl, get_package_details logging.getLogger().setLevel(logging.INFO) root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + +def get_package_wheel_path(pkg_root): + # parse setup.py to get package name and version + pkg_name, _, version = get_package_details(os.path.join(pkg_root, "setup.py")) + # Check if wheel is already built and available for current package + prebuilt_dir = os.getenv("PREBUILT_WHEEL_DIR") + if prebuilt_dir: + prebuilt_package_path = find_whl(prebuilt_dir, pkg_name, version) + else: + return None + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run apistubgen against target folder. ") @@ -36,14 +49,20 @@ args = parser.parse_args() + + # Check if a wheel is already built for current package and install from wheel when available + # If wheel is not available then install package from source + pkg_path = get_package_wheel_path(args.target_package) + if not pkg_path: + pkg_path = args.target_package + + check_call( [ "apistubgen", "--pkg-path", - args.target_package, + pkg_path, ], cwd=args.work_dir ) - - diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 994cb293dbef..52228b852047 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -233,6 +233,6 @@ deps = {[base]deps} commands = # install API stub generator - {envbindir}/python -m pip install "git+https://github.com/praveenkuttappan/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" + {envbindir}/python -m pip install "git+https://github.com/azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" {envbindir}/python -m pip freeze - apistubgen --pkg-path {toxinidir} --temp-path {envtmpdir} --out-path {envtmpdir} --verbose + {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} \ No newline at end of file From 6b14990593c59c75c87de243ca330db46bc182f9 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 01:42:23 -0700 Subject: [PATCH 08/14] add step to download artifacts --- eng/pipelines/templates/steps/run_apistub.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index a8691c39d809..0cde36c32486 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -16,6 +16,11 @@ steps: displayName: 'Prep Environment' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + - task: PythonScript@0 displayName: 'Run Api Stub Generation' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) From 8988893f9465e52a59e26662e0cba1414d20bd68 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 09:59:33 -0700 Subject: [PATCH 09/14] update python version to 3.8 --- eng/pipelines/templates/steps/run_apistub.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 0cde36c32486..21eff2226900 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -6,10 +6,10 @@ parameters: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' + displayName: 'Use Python 3.8' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) inputs: - versionSpec: '3.7' + versionSpec: '3.8' - script: | pip install -r eng/ci_tools.txt From 1385633f5d9b1c10072bb821b7f4355982b377cc Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 10:51:44 -0700 Subject: [PATCH 10/14] change vm image to windows --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 8124bca8ac2f..32edaa0d3433 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -65,7 +65,7 @@ jobs: - 'Build' pool: - vmImage: 'ubuntu-18.04' + vmImage: 'windows-2019' steps: - template: ../steps/analyze.yml From 0af310ed1c4381298071e4da13fa24da17d62470 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 11:27:29 -0700 Subject: [PATCH 11/14] change vm image to windows --- eng/tox/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 52228b852047..c279af3e0426 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -227,7 +227,7 @@ commands = [testenv:apistub] skipsdist = false skip_install = false -usedevelop = true +usedevelop = false changedir = {envtmpdir} deps = {[base]deps} From ac336a0a7d7ab4543bf3663555ab22d92aeabda6 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 11:28:11 -0700 Subject: [PATCH 12/14] revert vm image to windows --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 32edaa0d3433..8124bca8ac2f 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -65,7 +65,7 @@ jobs: - 'Build' pool: - vmImage: 'windows-2019' + vmImage: 'ubuntu-18.04' steps: - template: ../steps/analyze.yml From 66f2cf2c1dab0dfde187d853cf845b1eb89d3f84 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 11:53:49 -0700 Subject: [PATCH 13/14] Revert debug changes --- eng/pipelines/templates/steps/analyze.yml | 5 +++++ eng/pipelines/templates/steps/run_apistub.yml | 9 ++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 57d7b01a3d84..eae12578bda1 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -10,6 +10,11 @@ steps: inputs: versionSpec: '$(PythonVersion)' + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + - script: | pip install -r eng/ci_tools.txt ward scan -d $(Build.SourcesDirectory) -c $(Build.SourcesDirectory)/eng/.docsettings.yml diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 21eff2226900..a8691c39d809 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -6,21 +6,16 @@ parameters: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.8' + displayName: 'Use Python 3.7' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) inputs: - versionSpec: '3.8' + versionSpec: '3.7' - script: | pip install -r eng/ci_tools.txt displayName: 'Prep Environment' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: 'artifacts' - targetPath: $(Build.ArtifactStagingDirectory) - - task: PythonScript@0 displayName: 'Run Api Stub Generation' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) From 2c48ba6f7d6cb16b8f76f40da5b1fcad4f69c610 Mon Sep 17 00:00:00 2001 From: praveenkuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 22 Jul 2020 12:10:02 -0700 Subject: [PATCH 14/14] Formatting changes --- eng/tox/run_apistubgen.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/eng/tox/run_apistubgen.py b/eng/tox/run_apistubgen.py index c279f42c61c9..0e37243cd16f 100644 --- a/eng/tox/run_apistubgen.py +++ b/eng/tox/run_apistubgen.py @@ -4,11 +4,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. -from subprocess import check_call, CalledProcessError +from subprocess import check_call import argparse import os import logging -import sys from tox_helper_tasks import find_whl, get_package_details @@ -29,7 +28,9 @@ def get_package_wheel_path(pkg_root): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run apistubgen against target folder. ") + parser = argparse.ArgumentParser( + description="Run apistubgen against target folder. " + ) parser.add_argument( "-t", @@ -47,7 +48,6 @@ def get_package_wheel_path(pkg_root): required=True, ) - args = parser.parse_args() # Check if a wheel is already built for current package and install from wheel when available @@ -56,13 +56,4 @@ def get_package_wheel_path(pkg_root): if not pkg_path: pkg_path = args.target_package - - check_call( - [ - "apistubgen", - "--pkg-path", - pkg_path, - ], - - cwd=args.work_dir - ) + check_call(["apistubgen", "--pkg-path", pkg_path,], cwd=args.work_dir)