From 55b88ca44fc5e46fb7a14b18f92f64d21fa5517a Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:13:20 +0000 Subject: [PATCH 01/33] artifact --- tests/ci/azure_artifact_feed.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/ci/azure_artifact_feed.yaml diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml new file mode 100644 index 0000000000..b38d7b3acd --- /dev/null +++ b/tests/ci/azure_artifact_feed.yaml @@ -0,0 +1,3 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + From 40666ad45420e84ebe388ffabc90723bd78b9fde Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:24:43 +0000 Subject: [PATCH 02/33] artifact --- tests/ci/azure_artifact_feed.yaml | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index b38d7b3acd..8d602ad7f5 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -1,3 +1,60 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +trigger: +- master + +variables: +- group: LinuxAgentPool + +jobs: +- job: Artifact + displayName: 'Create Recommenders artifact' + timeoutInMinutes: 30 # how long to run the job before automatically cancelling + pool: + name: $(Agent_Pool) + + steps: + - bash: | + echo "##vso[task.prependpath]/data/anaconda/bin" + conda env list + displayName: 'Add Conda to PATH' + + - script: | + conda env remove -n artifact_reco_base -y + workingDirectory: tests + displayName: 'Conda remove' + continueOnError: true + condition: always() # this step will always run, even if the pipeline is canceled + + - script: | + python ./scripts/generate_conda_file.py --name artifact_reco_base + conda env create --quiet -f artifact_reco_base.yaml 2> log + displayName: 'Setup Conda Env' + + - script: | + conda activate artifact_reco_base + pip install wheel twine keyring artifacts-keyring + pip list + displayName: 'Install dependencies' + + - script: | + python setup.py sdist bdist_wheel + displayName: 'Build wheel' + + - task: TwineAuthenticate@1 + inputs: + artifactFeed: best-practices/recommenders + displayName: 'Twine Authenticate' + + - script: | + cat $(PYPIRC_PATH) + python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose + displayName: 'Upload wheel' + + - script: | + conda env remove -n artifact_reco_base -y + workingDirectory: tests + displayName: 'Conda remove' + continueOnError: true + condition: always() # this step will always run, even if the pipeline is canceled From ce45342ae886d6790e74d2669be3774312b98d66 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:36:52 +0000 Subject: [PATCH 03/33] :bug: --- tests/ci/azure_artifact_feed.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 8d602ad7f5..8953b3574f 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -33,8 +33,9 @@ jobs: displayName: 'Setup Conda Env' - script: | - conda activate artifact_reco_base - pip install wheel twine keyring artifacts-keyring + . /anaconda/etc/profile.d/conda.sh && \ + conda activate artifact_reco_base && \ + pip install wheel twine keyring artifacts-keyring && \ pip list displayName: 'Install dependencies' @@ -48,7 +49,7 @@ jobs: displayName: 'Twine Authenticate' - script: | - cat $(PYPIRC_PATH) + python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose displayName: 'Upload wheel' From 2ae7a88017e871262ead93cfaf70e7be493adca3 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:37:32 +0000 Subject: [PATCH 04/33] trigger --- tests/ci/azure_artifact_feed.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 8953b3574f..832c7d394d 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -1,8 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -trigger: -- master +#trigger: +#- master variables: - group: LinuxAgentPool From 09eb6d60107b73bd2765a9166e0ffc63560ad161 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:40:53 +0000 Subject: [PATCH 05/33] :bug: --- tests/ci/azure_artifact_feed.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 832c7d394d..4aeb62cebb 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -40,6 +40,8 @@ jobs: displayName: 'Install dependencies' - script: | + . /anaconda/etc/profile.d/conda.sh && \ + conda activate artifact_reco_base && \ python setup.py sdist bdist_wheel displayName: 'Build wheel' @@ -49,7 +51,8 @@ jobs: displayName: 'Twine Authenticate' - script: | - + . /anaconda/etc/profile.d/conda.sh && \ + conda activate artifact_reco_base && \ python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose displayName: 'Upload wheel' From ba33f2fff04087128082119604fcbf3783d2c1ae Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 11:54:02 +0000 Subject: [PATCH 06/33] :bug: --- tests/ci/azure_artifact_feed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 4aeb62cebb..b9a38333f7 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -47,7 +47,7 @@ jobs: - task: TwineAuthenticate@1 inputs: - artifactFeed: best-practices/recommenders + artifactFeed: recommenders/recommenders displayName: 'Twine Authenticate' - script: | From 93291d7dbd18453c9153098bfc5a22468e66f88f Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 12:20:57 +0000 Subject: [PATCH 07/33] fix setup --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a603346f8f..41810f6d54 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from setuptools import setup +from setuptools import setup, find_packages from os import chdir, path chdir(path.abspath(path.dirname(__file__))) @@ -30,6 +30,7 @@ "Programming Language :: Python :: 3.6", ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", - packages=["reco_utils"], + package_dir={"": "reco_utils"}, + packages=find_packages([where="reco_utils"], exclude=["azureml_designer_modules"]) python_requires=">=3.6, <4", ) From c999d0fc30be20919cc9c85050dedbf0aef481a9 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 12:26:55 +0000 Subject: [PATCH 08/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 41810f6d54..4bc5cd0074 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,6 @@ ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", package_dir={"": "reco_utils"}, - packages=find_packages([where="reco_utils"], exclude=["azureml_designer_modules"]) + packages=find_packages(where="reco_utils", exclude=["azureml_designer_modules"]) python_requires=">=3.6, <4", ) From 4b461041317f90f8a44a4d755e781c98747860c3 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 12:27:36 +0000 Subject: [PATCH 09/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4bc5cd0074..d55a101092 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,6 @@ ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", package_dir={"": "reco_utils"}, - packages=find_packages(where="reco_utils", exclude=["azureml_designer_modules"]) + packages=find_packages(where="reco_utils", exclude=["azureml_designer_modules"]), python_requires=">=3.6, <4", ) From 7099a1a3b642d57e09df0090ae325cee02a76983 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 10 Mar 2020 15:42:46 +0000 Subject: [PATCH 10/33] version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d55a101092..af1a59c77b 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( name="reco_utils", - version=VERSION, + version=VERSION + ".post0", description="Recommender System Utilities", long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", From ac25c9b1da61676da396f2252de76846028f7bd4 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 10:47:32 +0000 Subject: [PATCH 11/33] =?UTF-8?q?=C3=B1apa=20inspired=20by=20the=20great?= =?UTF-8?q?=20@eisber?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 10 +++++++--- tests/ci/azure_artifact_feed.yaml | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index af1a59c77b..ef7d89f92c 100644 --- a/setup.py +++ b/setup.py @@ -2,18 +2,22 @@ # Licensed under the MIT License. from setuptools import setup, find_packages -from os import chdir, path +from os import chdir, path, environ chdir(path.abspath(path.dirname(__file__))) -VERSION = __import__("reco_utils.__init__").VERSION +version = __import__("reco_utils.__init__").VERSION # Get the long description from the README file with open(path.join("reco_utils", "README.md"), encoding="utf-8") as f: LONG_DESCRIPTION = f.read() +HASH = environ.get("HASH", "") +if HASH: + version += "." + HASH + setup( name="reco_utils", - version=VERSION + ".post0", + version=version, description="Recommender System Utilities", long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index b9a38333f7..e4f8eb9986 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -42,7 +42,8 @@ jobs: - script: | . /anaconda/etc/profile.d/conda.sh && \ conda activate artifact_reco_base && \ - python setup.py sdist bdist_wheel + rm -rf dist && \ + HASH=`git log --pretty=format:'%h' -n 1` python setup.py sdist bdist_wheel displayName: 'Build wheel' - task: TwineAuthenticate@1 From a3bd56d1daba50fd97b1a918119c7f8b83aa515e Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 10:55:24 +0000 Subject: [PATCH 12/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ef7d89f92c..e9c859f40a 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ HASH = environ.get("HASH", "") if HASH: - version += "." + HASH + version += ".dev" + HASH setup( name="reco_utils", From 9c353ded5af08c0c401c05563c6abd84a47b6be5 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 10:59:22 +0000 Subject: [PATCH 13/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e9c859f40a..7b95e0af9c 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ HASH = environ.get("HASH", "") if HASH: - version += ".dev" + HASH + version += ".post" + HASH setup( name="reco_utils", From 5c6d0288fa93403935b829afb5b5fed55ef718df Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 11:03:52 +0000 Subject: [PATCH 14/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7b95e0af9c..f82501a83d 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ HASH = environ.get("HASH", "") if HASH: - version += ".post" + HASH + version += ".dev0" + HASH setup( name="reco_utils", From e643f5d71c401c5c7f970e71e1d7173210b855de Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 11:14:16 +0000 Subject: [PATCH 15/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f82501a83d..18f7afc404 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ HASH = environ.get("HASH", "") if HASH: - version += ".dev0" + HASH + version += ".dev0+" + HASH setup( name="reco_utils", From 65b13d5f03cafaba1b7134f5ae28299dda5bf2c1 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 11:24:09 +0000 Subject: [PATCH 16/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 18f7afc404..cfae279fb0 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="reco_utils", - version=version, + version="0.0.dev0+" + HASH, description="Recommender System Utilities", long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", From e0b3316f26249d95be4230fcdde0c6fea96d9bcc Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 12:01:57 +0000 Subject: [PATCH 17/33] :bug: --- setup.py | 9 +++++---- tests/ci/azure_artifact_feed.yaml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index cfae279fb0..52d8e19830 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import time from setuptools import setup, find_packages from os import chdir, path, environ @@ -11,13 +12,13 @@ with open(path.join("reco_utils", "README.md"), encoding="utf-8") as f: LONG_DESCRIPTION = f.read() -HASH = environ.get("HASH", "") -if HASH: - version += ".dev0+" + HASH +HASH = environ.get("HASH", None) +if HASH is not None: + version += ".dev" + str(int(time.time())) setup( name="reco_utils", - version="0.0.dev0+" + HASH, + version=version, description="Recommender System Utilities", long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index e4f8eb9986..9112ad13ca 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -43,7 +43,7 @@ jobs: . /anaconda/etc/profile.d/conda.sh && \ conda activate artifact_reco_base && \ rm -rf dist && \ - HASH=`git log --pretty=format:'%h' -n 1` python setup.py sdist bdist_wheel + HASH=True python setup.py sdist bdist_wheel displayName: 'Build wheel' - task: TwineAuthenticate@1 From 518845f98a04f5c72590d08c04cedd5963803cd8 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 12:06:12 +0000 Subject: [PATCH 18/33] :bug: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 52d8e19830..ec1194b338 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ HASH = environ.get("HASH", None) if HASH is not None: - version += ".dev" + str(int(time.time())) + version += ".post" + str(int(time.time())) setup( name="reco_utils", From 59db97747fe09a1eebbce5bb859c932b4dccec3b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 11 Mar 2020 12:10:49 +0000 Subject: [PATCH 19/33] :bug: --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ec1194b338..77e5f5b229 100644 --- a/setup.py +++ b/setup.py @@ -39,3 +39,4 @@ packages=find_packages(where="reco_utils", exclude=["azureml_designer_modules"]), python_requires=">=3.6, <4", ) + From 4638e289fe7da9e82c8f507a93ddc51092393ebd Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 11:15:44 +0000 Subject: [PATCH 20/33] env --- setup.py | 4 +++- tests/ci/azure_artifact_feed.yaml | 32 ++++++++++++++++--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 77e5f5b229..55e3dafbd5 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,10 @@ if HASH is not None: version += ".post" + str(int(time.time())) +name = environ.get("NAME", "reco_utils") + setup( - name="reco_utils", + name=name, version=version, description="Recommender System Utilities", long_description=LONG_DESCRIPTION, diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 9112ad13ca..49d8caf37b 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -4,6 +4,8 @@ #trigger: #- master +# The global variables are defined in ADO/Pipelines/Library +# https://dev.azure.com/best-practices/recommenders/_library?itemType=VariableGroups variables: - group: LinuxAgentPool @@ -21,44 +23,44 @@ jobs: displayName: 'Add Conda to PATH' - script: | - conda env remove -n artifact_reco_base -y + conda env remove -n $(env_name) -y workingDirectory: tests displayName: 'Conda remove' continueOnError: true condition: always() # this step will always run, even if the pipeline is canceled - script: | - python ./scripts/generate_conda_file.py --name artifact_reco_base - conda env create --quiet -f artifact_reco_base.yaml 2> log + python ./scripts/generate_conda_file.py --name $(env_name) + conda env create --quiet -f $(env_name).yaml 2> log displayName: 'Setup Conda Env' - script: | . /anaconda/etc/profile.d/conda.sh && \ - conda activate artifact_reco_base && \ + conda activate $(env_name) && \ pip install wheel twine keyring artifacts-keyring && \ pip list displayName: 'Install dependencies' - script: | . /anaconda/etc/profile.d/conda.sh && \ - conda activate artifact_reco_base && \ + conda activate $(env_name) && \ rm -rf dist && \ HASH=True python setup.py sdist bdist_wheel displayName: 'Build wheel' - - task: TwineAuthenticate@1 - inputs: - artifactFeed: recommenders/recommenders - displayName: 'Twine Authenticate' + # - task: TwineAuthenticate@1 + # inputs: + # artifactFeed: recommenders/recommenders + # displayName: 'Twine Authenticate' - - script: | - . /anaconda/etc/profile.d/conda.sh && \ - conda activate artifact_reco_base && \ - python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose - displayName: 'Upload wheel' + # - script: | + # . /anaconda/etc/profile.d/conda.sh && \ + # conda activate $(env_name) && \ + # python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose + # displayName: 'Upload wheel' - script: | - conda env remove -n artifact_reco_base -y + conda env remove -n $(env_name) -y workingDirectory: tests displayName: 'Conda remove' continueOnError: true From 0ab62d2c2db42eb8edf57f5a8b690887da12bd56 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 11:32:36 +0000 Subject: [PATCH 21/33] variables --- tests/ci/azure_artifact_feed.yaml | 36 ++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 49d8caf37b..dcfa8ebc4b 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -1,14 +1,24 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -#trigger: -#- master +# NOTE: +# This yaml file is used to generate ADO artifacts for Recommenders library. +# We use the same yaml file to generate the library in different environments: CPU, GPU and PySpark +# When setting the pipeline, we need to define the variables env_name, env_flag and library_name +# To add the variables, go to the pipeline, press edit and then variables. +# An example of the variables we use in the GPU environment: +# env_name=artifact_reco_gpu +# env_flag=--gpu +# library_name=reco_utils_gpu # The global variables are defined in ADO/Pipelines/Library # https://dev.azure.com/best-practices/recommenders/_library?itemType=VariableGroups variables: - group: LinuxAgentPool +#trigger: +#- master + jobs: - job: Artifact displayName: 'Create Recommenders artifact' @@ -30,7 +40,7 @@ jobs: condition: always() # this step will always run, even if the pipeline is canceled - script: | - python ./scripts/generate_conda_file.py --name $(env_name) + python ./scripts/generate_conda_file.py --name $(env_name) $(env_flag) conda env create --quiet -f $(env_name).yaml 2> log displayName: 'Setup Conda Env' @@ -45,19 +55,19 @@ jobs: . /anaconda/etc/profile.d/conda.sh && \ conda activate $(env_name) && \ rm -rf dist && \ - HASH=True python setup.py sdist bdist_wheel + HASH=True NAME=$(library_name) python setup.py sdist bdist_wheel displayName: 'Build wheel' - # - task: TwineAuthenticate@1 - # inputs: - # artifactFeed: recommenders/recommenders - # displayName: 'Twine Authenticate' + - task: TwineAuthenticate@1 + inputs: + artifactFeed: recommenders/recommenders + displayName: 'Twine Authenticate' - # - script: | - # . /anaconda/etc/profile.d/conda.sh && \ - # conda activate $(env_name) && \ - # python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose - # displayName: 'Upload wheel' + - script: | + . /anaconda/etc/profile.d/conda.sh && \ + conda activate $(env_name) && \ + python -m twine upload -r recommenders --config-file $(PYPIRC_PATH) dist/*.whl --verbose + displayName: 'Upload wheel' - script: | conda env remove -n $(env_name) -y From 7846420a941020c6fa9e1aa9e601d29fd5aa1b82 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 11:34:11 +0000 Subject: [PATCH 22/33] :boom: --- tests/ci/azure_artifact_feed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index dcfa8ebc4b..8a1b0877dc 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -63,6 +63,7 @@ jobs: artifactFeed: recommenders/recommenders displayName: 'Twine Authenticate' + # Artifact package: https://dev.azure.com/best-practices/recommenders/_packaging?_a=feed&feed=recommenders - script: | . /anaconda/etc/profile.d/conda.sh && \ conda activate $(env_name) && \ From ee38052755d3968a456e5441fac406ffd8387e2b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 11:59:22 +0000 Subject: [PATCH 23/33] trigger --- tests/ci/azure_artifact_feed.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 8a1b0877dc..965654593c 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -16,8 +16,8 @@ variables: - group: LinuxAgentPool -#trigger: -#- master +trigger: +- master jobs: - job: Artifact From 1d3fa81ae0f7e9791020c2762353e7c0f2827c0a Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 12:09:29 +0000 Subject: [PATCH 24/33] :memo: --- tests/ci/azure_artifact_feed.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 965654593c..63361b7d77 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -1,6 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# To install recommenders artifact: +# $ pip install artifacts-keyring --pre +# Choose the version to install based on your environment: +# $ pip install reco-utils --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ +# $ pip install reco-utils-gpu --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ +# $ pip install reco-utils-pyspark --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ + + # NOTE: # This yaml file is used to generate ADO artifacts for Recommenders library. # We use the same yaml file to generate the library in different environments: CPU, GPU and PySpark From ed11a60e0035a45e0a537c570a3877f3106a574e Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 12 Mar 2020 12:47:06 +0000 Subject: [PATCH 25/33] :bug: --- setup.py | 3 +-- tests/ci/azure_artifact_feed.yaml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 55e3dafbd5..6ab4197b52 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,7 @@ "Programming Language :: Python :: 3.6", ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", - package_dir={"": "reco_utils"}, - packages=find_packages(where="reco_utils", exclude=["azureml_designer_modules"]), + packages=["reco_utils"], python_requires=">=3.6, <4", ) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 63361b7d77..c874a1222a 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -24,8 +24,8 @@ variables: - group: LinuxAgentPool -trigger: -- master +# trigger: +# - master jobs: - job: Artifact From 6d11712bfb640a7d2c89fd7ef9eedc63c317149d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 16 Mar 2020 16:02:35 +0000 Subject: [PATCH 26/33] :bug: --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ab4197b52..3d18e0bfb1 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ "Programming Language :: Python :: 3.6", ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", - packages=["reco_utils"], + package_dir={"": "reco_utils"}, + packages=find_packages(where="reco_utils", exclude=["tests", "scripts"]), python_requires=">=3.6, <4", ) From 099b05a956c1dbedcccf2ad0b2fb158e99e8b328 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 17 Mar 2020 09:42:37 +0000 Subject: [PATCH 27/33] executed when PR to master --- tests/ci/azure_artifact_feed.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index c874a1222a..9277ab3d26 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -24,8 +24,13 @@ variables: - group: LinuxAgentPool -# trigger: -# - master +# Pull request against these branches will trigger this build +pr: +- master + +# Any commit to this branch will trigger the build. +trigger: +- master jobs: - job: Artifact From 1287a2df88144a61b480523b3486c1358d937633 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 18 Mar 2020 12:39:57 +0000 Subject: [PATCH 28/33] package_dir --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3d18e0bfb1..f0d63c1c51 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,8 @@ "Programming Language :: Python :: 3.6", ], keywords="recommendations recommenders recommender system engine machine learning python spark gpu", - package_dir={"": "reco_utils"}, - packages=find_packages(where="reco_utils", exclude=["tests", "scripts"]), + package_dir={"reco_utils": "reco_utils"}, + packages=find_packages(where=".", exclude=["tests", "scripts"]), python_requires=">=3.6, <4", ) From 089f7a1aff3bbcb582a1422d7378c44186b584d9 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 18 Mar 2020 12:45:25 +0000 Subject: [PATCH 29/33] :bug: the correct format is 2019.9 instead of 2019.09 --- reco_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reco_utils/__init__.py b/reco_utils/__init__.py index 19ce5b102a..df052a7fad 100644 --- a/reco_utils/__init__.py +++ b/reco_utils/__init__.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. __title__ = "Microsoft Recommenders" -__version__ = "2019.09" +__version__ = "2019.9" __author__ = "RecoDev Team at Microsoft" __license__ = "MIT" __copyright__ = "Copyright 2018-present Microsoft Corporation" From 504b8d69e7b51688afb5b6fa6b9fbb145849fc4b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 18 Mar 2020 16:02:08 +0000 Subject: [PATCH 30/33] ado artifacts need pip>=19.2 --- scripts/generate_conda_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_conda_file.py b/scripts/generate_conda_file.py index d8e791918c..7f33e97b9e 100644 --- a/scripts/generate_conda_file.py +++ b/scripts/generate_conda_file.py @@ -45,7 +45,7 @@ "mock": "mock==2.0.0", "numpy": "numpy>=1.13.3", "pandas": "pandas>=0.23.4,<1.0.0", - "pip": "pip>=19.0.3", + "pip": "pip>=19.2", "pytest": "pytest>=3.6.4", "pytorch": "pytorch-cpu>=1.0.0", "seaborn": "seaborn>=0.8.1", @@ -84,7 +84,7 @@ "nbconvert": "nbconvert==5.5.0", "pydocumentdb": "pydocumentdb>=2.3.3", "pymanopt": "pymanopt==0.2.3", - "xlearn": "xlearn==0.40a1" + "xlearn": "xlearn==0.40a1", } PIP_GPU = {"nvidia-ml-py3": "nvidia-ml-py3>=7.352.0"} From 3330ef367c0420fb007e755c5202bc50c163c77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= <3491412+miguelgfierro@users.noreply.github.com> Date: Thu, 19 Mar 2020 10:52:12 +0000 Subject: [PATCH 31/33] execute when merging to master, great advice by @atbagga --- tests/ci/azure_artifact_feed.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 9277ab3d26..3abc62a28e 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -24,11 +24,8 @@ variables: - group: LinuxAgentPool -# Pull request against these branches will trigger this build -pr: -- master - -# Any commit to this branch will trigger the build. +# The pipeline will be triggered when we merge to master +pr: none trigger: - master From 4aef8f6eff085f4203f37d3c7fa5bdafa781c6d0 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 23 Mar 2020 10:10:19 +0000 Subject: [PATCH 32/33] :memo: --- tests/ci/azure_artifact_feed.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 3abc62a28e..6133dc8f3f 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -1,13 +1,24 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# To install recommenders artifact: +# To install recommenders artifact (with manual login to Azure): # $ pip install artifacts-keyring --pre # Choose the version to install based on your environment: # $ pip install reco-utils --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ # $ pip install reco-utils-gpu --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ # $ pip install reco-utils-pyspark --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ +# To install recommenders artifact programmarically with Personal Access Token (PAT): +# First generate the PAT: +# 1. Go to the Personal Access Tokens screen in ADO => User (icon in top right) +# 2. Generate a new token, with at least Packages/Read permissions selected +# 3. Add the PAT in the index-url=https://{feed_name}:{PAT}@pkgs.dev.azure.com/{org_name}/{project_name}/_packaging/{feed_name}/pypi/simple/ +# Install dependencies and Recommenders library +# $ pip install artifacts-keyring --pre +# Choose the version to install based on your environment: +# $ pip install reco-utils --index-url=https://recommenders:{PAT}@pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ +# $ pip install reco-utils-gpu --index-url=https://recommenders:{PAT}@pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ +# $ pip install reco-utils-pyspark --index-url=https://recommenders:{PAT}@pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ # NOTE: # This yaml file is used to generate ADO artifacts for Recommenders library. From 92a73459d73743cb13970dce6d22d254eba20459 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 23 Mar 2020 10:17:01 +0000 Subject: [PATCH 33/33] :memo: --- tests/ci/azure_artifact_feed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azure_artifact_feed.yaml b/tests/ci/azure_artifact_feed.yaml index 6133dc8f3f..4260ae3a02 100644 --- a/tests/ci/azure_artifact_feed.yaml +++ b/tests/ci/azure_artifact_feed.yaml @@ -8,7 +8,7 @@ # $ pip install reco-utils-gpu --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ # $ pip install reco-utils-pyspark --index-url=https://pkgs.dev.azure.com/best-practices/recommenders/_packaging/recommenders/pypi/simple/ -# To install recommenders artifact programmarically with Personal Access Token (PAT): +# To install recommenders artifact programmatically with Personal Access Token (PAT): # First generate the PAT: # 1. Go to the Personal Access Tokens screen in ADO => User (icon in top right) # 2. Generate a new token, with at least Packages/Read permissions selected