From b3abc8ad1ad26ca6277501af8e81dd483253c28d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 15 May 2024 06:53:39 +0200 Subject: [PATCH 01/29] Enabling python 3.12 #2097 Signed-off-by: miguelgfierro --- .github/workflows/azureml-cpu-nightly.yml | 2 +- .github/workflows/azureml-gpu-nightly.yml | 2 +- .github/workflows/azureml-spark-nightly.yml | 2 +- .github/workflows/azureml-unit-tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 93e414564..0f74cde71 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"'] + python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"', '"python=3.12"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 3b9f6d6b4..1b8108eda 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"'] + python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"', '"python=3.12"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index 8f28be6f2..e9364cfc0 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -66,7 +66,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"'] + python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"', '"python=3.12"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index b39268318..6dbde7fe8 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -56,7 +56,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"'] + python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"', '"python=3.12"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code From 809a1d39c3e4bdab72aa56ee80d9eb79413ac29d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 15 May 2024 06:54:03 +0200 Subject: [PATCH 02/29] Enabling python 3.12 #2097 Signed-off-by: miguelgfierro --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index db4e1012b..601704610 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: POSIX :: Linux", ], extras_require=extras_require, From 496b22fca0faf861ff98739c8deec46b031e8d44 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 15 May 2024 07:19:07 +0200 Subject: [PATCH 03/29] Cornac>2 Signed-off-by: miguelgfierro --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 601704610..64108d179 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ install_requires = [ "category-encoders>=2.6.0,<3", # requires packaging - "cornac>=1.15.2,<2", # requires packaging, tqdm + "cornac>=1.15.2,<3", # requires packaging, tqdm "hyperopt>=0.2.7,<1", "lightfm>=1.17,<2", # requires requests "lightgbm>=4.0.0,<5", From 2b9ddcc4d91d0c43a89263ad0275167606fa1e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 15 May 2024 10:39:44 +0200 Subject: [PATCH 04/29] setuptools >= 67 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0ff4c8d96..18574aba5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [build-system] requires = [ - "setuptools>=52", + "setuptools>=67", "wheel>=0.36", "numpy>=1.15", ] dependencies = [ - "setuptools>=52", + "setuptools>=67", "wheel>=0.36", "numpy>=1.15", ] @@ -17,4 +17,4 @@ markers = [ "gpu: tests running on GPU", "notebooks: tests for notebooks", "spark: tests that requires Spark", -] \ No newline at end of file +] From 69f2d36f3ce12bce6fe58246cd6cfce5e15cb617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Tue, 21 May 2024 17:59:14 +0200 Subject: [PATCH 05/29] Temporary use lightfm patch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 64108d179..a62eda445 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ "category-encoders>=2.6.0,<3", # requires packaging "cornac>=1.15.2,<3", # requires packaging, tqdm "hyperopt>=0.2.7,<1", - "lightfm>=1.17,<2", # requires requests + # TODO: Wait for the PR to be merged and released + "lightfm@git+https://github.com/daviddavo/lightfm", # requires requests "lightgbm>=4.0.0,<5", "locust>=2.12.2,<3", # requires jinja2 "memory-profiler>=0.61.0,<1", From c223e3de85b216f9267a11afc20b74195c945537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 13:04:22 +0200 Subject: [PATCH 06/29] Updated tensorflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc72fec95..b16f12cf4 100644 --- a/setup.py +++ b/setup.py @@ -52,8 +52,9 @@ "gpu": [ "fastai>=2.7.11,<3", "nvidia-ml-py>=11.525.84", - "tensorflow>=2.8.4,!=2.9.0.*,!=2.9.1,!=2.9.2,!=2.10.0.*,<2.16", # Fixed TF due to constant security problems and breaking changes #2073 + "tensorflow>=2.8.4,!=2.9.0.*,!=2.9.1,!=2.9.2,!=2.10.0.*,<2.17", # Fixed TF due to constant security problems and breaking changes #2073 "tf-slim>=1.1.0", # No python_requires in its setup.py + "tf-keras<3", "torch>=2.0.1,<3", ], "spark": [ From 75b884b382986cbcf73897a0e8b7059512d00104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 13:31:55 +0200 Subject: [PATCH 07/29] Update pymanopt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- recommenders/README.md | 2 +- setup.py | 2 +- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/recommenders/README.md b/recommenders/README.md index 70324d9de..a8a0206f9 100644 --- a/recommenders/README.md +++ b/recommenders/README.md @@ -76,7 +76,7 @@ We are currently evaluating inclusion of the following dependencies: Some dependencies are not available via the recommenders PyPI package, but can be installed in the following ways: - pymanopt: this dependency is required for the RLRMC and GeoIMC algorithms; a version of this code compatible with TensorFlow 2 can be - installed with `pip install "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip"`. + installed with `pip install "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip"`. ## NNI dependencies diff --git a/setup.py b/setup.py index b16f12cf4..0dd7fa7d2 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ # The following dependency can be installed as below, however PyPI does not allow direct URLs. # Temporary fix for pymanopt, only this commit works with TF2 -# "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip", +# "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip", setup( name="recommenders", diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index adda7e172..309d3c5b6 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -221,7 +221,7 @@ def create_run_config( conda_dep = CondaDependencies() conda_dep.add_conda_package(conda_pkg_python) conda_dep.add_pip_package( - "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip" + "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip" ) # install recommenders From 0c28cf11c327e238ee3caa5234bb992259e6226d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 13:40:12 +0200 Subject: [PATCH 08/29] Installing pymanopt from git MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- recommenders/README.md | 2 +- setup.py | 2 +- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/recommenders/README.md b/recommenders/README.md index a8a0206f9..6861a2e6b 100644 --- a/recommenders/README.md +++ b/recommenders/README.md @@ -76,7 +76,7 @@ We are currently evaluating inclusion of the following dependencies: Some dependencies are not available via the recommenders PyPI package, but can be installed in the following ways: - pymanopt: this dependency is required for the RLRMC and GeoIMC algorithms; a version of this code compatible with TensorFlow 2 can be - installed with `pip install "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip"`. + installed with `pip install "git+https://github.com/pymanopt/pymanopt.git@1de3b6f47258820fdc072fceaeaa763b9fd263b0"`. ## NNI dependencies diff --git a/setup.py b/setup.py index 0dd7fa7d2..5e1f2436c 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ # The following dependency can be installed as below, however PyPI does not allow direct URLs. # Temporary fix for pymanopt, only this commit works with TF2 -# "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip", +# "git+https://github.com/pymanopt/pymanopt.git@1de3b6f47258820fdc072fceaeaa763b9fd263b0", setup( name="recommenders", diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 309d3c5b6..61121a90a 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -221,7 +221,7 @@ def create_run_config( conda_dep = CondaDependencies() conda_dep.add_conda_package(conda_pkg_python) conda_dep.add_pip_package( - "pymanopt@https://github.com/pymanopt/pymanopt/archive/1de3b6f47258820fdc072fceaeaa763b9fd263b0.zip" + "git+https://github.com/pymanopt/pymanopt.git@1de3b6f47258820fdc072fceaeaa763b9fd263b0" ) # install recommenders From ad0c79d327426af7994d94154cdc1593aac9f449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 15:10:05 +0200 Subject: [PATCH 09/29] Add setuptools (for python 3.12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- .github/actions/azureml-test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 85ae9f84a..43c1ebf79 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -79,9 +79,9 @@ runs: uses: azure/login@v2 with: creds: ${{inputs.AZUREML_TEST_CREDENTIALS}} - - name: Install wheel package + - name: Install wheel and setuptools package shell: bash - run: pip install --quiet wheel + run: pip install --quiet wheel setuptools - name: Submit tests to AzureML shell: bash run: >- From cdf95158da35262c2ea1cd9d56040acce50c9478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 16:00:48 +0200 Subject: [PATCH 10/29] Use numpy 1.26 on python 3.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5e1f2436c..7e5404ce6 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ "nltk>=3.8.1,<4", # requires tqdm "notebook>=6.5.5,<8", # requires ipykernel, jinja2, jupyter, nbconvert, nbformat, packaging, requests "numba>=0.57.0,<1", + "numpy>=1.26;python_version>='3.12'", "pandas>2.0.0,<3.0.0", # requires numpy "pandera[strategies]>=0.6.5,<0.18;python_version<='3.8'", # For generating fake datasets "pandera[strategies]>=0.15.0;python_version>='3.9'", From a442edb576c64d5159f040bb9b81b6d89d2fae59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 3 Jul 2024 16:17:33 +0200 Subject: [PATCH 11/29] Update python version used to launch tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- .github/actions/azureml-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 43c1ebf79..6611f4275 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -71,7 +71,7 @@ runs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install azureml-core and azure-cli on a GitHub hosted server shell: bash run: pip install --quiet "azureml-core>1,<2" "azure-cli>2,<3" From dff4b0c08763f0dc23a331e93379e73553f989dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 5 Jul 2024 10:56:00 +0000 Subject: [PATCH 12/29] Added numpy 1.26 setup requires on python 3.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e5404ce6..f1d7c7715 100644 --- a/setup.py +++ b/setup.py @@ -126,5 +126,8 @@ where=".", exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], ), - setup_requires=["numpy>=1.19"], + setup_requires=[ + "numpy>=1.19", + "numpy>=1.26;python_version>='3.12'", + ], ) From 2af59a17f5eafc052e85066b164b591602f04cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 5 Jul 2024 11:14:24 +0000 Subject: [PATCH 13/29] [test ci] Force numpy 1.26 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f1d7c7715..ccaf4ff5f 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,6 @@ "nltk>=3.8.1,<4", # requires tqdm "notebook>=6.5.5,<8", # requires ipykernel, jinja2, jupyter, nbconvert, nbformat, packaging, requests "numba>=0.57.0,<1", - "numpy>=1.26;python_version>='3.12'", "pandas>2.0.0,<3.0.0", # requires numpy "pandera[strategies]>=0.6.5,<0.18;python_version<='3.8'", # For generating fake datasets "pandera[strategies]>=0.15.0;python_version>='3.9'", @@ -127,7 +126,6 @@ exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], ), setup_requires=[ - "numpy>=1.19", - "numpy>=1.26;python_version>='3.12'", + "numpy>=1.26", ], ) From a7208e8d0b4c49cf00f2c37000bf71822057739a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Thu, 18 Jul 2024 13:43:20 +0200 Subject: [PATCH 14/29] Updated conda in azureml tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 796f38dcc..9ef7ec3ef 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -199,12 +199,12 @@ def create_run_config( # Conda Environment # Pin pip=20.1.1 due to the issue: No module named 'ruamel' # See https://learn.microsoft.com/en-us/python/api/overview/azure/ml/install?view=azure-ml-py#troubleshooting -ENV MINICONDA_VERSION py38_23.3.1-0 +ENV MINICONDA_VERSION py310_24.5.0-0 ENV PATH /opt/miniconda/bin:$PATH -ENV CONDA_PACKAGE 23.5.0 +ENV CONDA_PACKAGE 24.5.0 RUN wget -qO /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ bash /tmp/miniconda.sh -bf -p /opt/miniconda && \ - conda install -y conda=${CONDA_PACKAGE} python=3.8 pip=20.1.1 && \ + conda install -y conda=${CONDA_PACKAGE} python=3.10 pip=24 && \ conda update --all -c conda-forge -y && \ conda clean -ay && \ rm -rf /opt/miniconda/pkgs && \ From 94ccdeb62260394cad908761d2521526e278751b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Thu, 18 Jul 2024 16:40:49 +0200 Subject: [PATCH 15/29] Update pymanopt usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- .../models/geoimc/geoimc_algorithm.py | 58 ++++++++++--------- tests/unit/recommenders/models/test_geoimc.py | 8 +-- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/recommenders/models/geoimc/geoimc_algorithm.py b/recommenders/models/geoimc/geoimc_algorithm.py index 04addf3f6..c67e3e7e8 100644 --- a/recommenders/models/geoimc/geoimc_algorithm.py +++ b/recommenders/models/geoimc/geoimc_algorithm.py @@ -11,8 +11,9 @@ from numba import njit, prange from pymanopt import Problem from pymanopt.manifolds import Stiefel, Product, SymmetricPositiveDefinite -from pymanopt.solvers import ConjugateGradient -from pymanopt.solvers.linesearch import LineSearchBackTracking +from pymanopt.autodiff.backends import numpy as backend_decorator +from pymanopt.optimizers import ConjugateGradient +from pymanopt.optimizers.line_search import BackTrackingLineSearcher class IMCProblem(object): @@ -68,7 +69,7 @@ def _computeLoss_csrmatrix(a, b, cd, indices, indptr, residual_global): residual_global[j] = num - cd[j] return residual_global - def _cost(self, params, residual_global): + def _cost(self, U, S, VT, residual_global): """Compute the cost of GeoIMC optimization problem Args: @@ -76,15 +77,11 @@ def _cost(self, params, residual_global): the cost needs to be evaluated. residual_global (csr_matrix): Residual matrix. """ - U = params[0] - B = params[1] - V = params[2] - - regularizer = 0.5 * self.lambda1 * np.sum(B**2) + regularizer = 0.5 * self.lambda1 * np.sum(S**2) IMCProblem._computeLoss_csrmatrix( - self.X.dot(U.dot(B)), - V.T.dot(self.Z.T), + self.X.dot(U.dot(S)), + VT.T.dot(self.Z.T), self.Y.data, self.Y.indices, self.Y.indptr, @@ -94,7 +91,7 @@ def _cost(self, params, residual_global): return cost - def _egrad(self, params, residual_global): + def _egrad(self, U, S, VT, residual_global): """Computes the euclidean gradient Args: @@ -102,29 +99,25 @@ def _egrad(self, params, residual_global): the cost needs to be evaluated. residual_global (csr_matrix): Residual matrix. """ - U = params[0] - B = params[1] - V = params[2] - residual_global_csr = csr_matrix( (residual_global, self.Y.indices, self.Y.indptr), shape=self.shape, ) gradU = ( - np.dot(self.X.T, residual_global_csr.dot(self.Z.dot(V.dot(B.T)))) + np.dot(self.X.T, residual_global_csr.dot(self.Z.dot(VT.dot(S.T)))) / self.nSamples ) gradB = ( - np.dot((self.X.dot(U)).T, residual_global_csr.dot(self.Z.dot(V))) + np.dot((self.X.dot(U)).T, residual_global_csr.dot(self.Z.dot(VT))) / self.nSamples - + self.lambda1 * B + + self.lambda1 * S ) gradB_sym = (gradB + gradB.T) / 2 gradV = ( - np.dot((self.X.dot(U.dot(B))).T, residual_global_csr.dot(self.Z)).T + np.dot((self.X.dot(U.dot(S))).T, residual_global_csr.dot(self.Z)).T / self.nSamples ) @@ -154,20 +147,29 @@ def _optimize(self, max_opt_time, max_opt_iter, verbosity): residual_global = np.zeros(self.Y.data.shape) solver = ConjugateGradient( - maxtime=max_opt_time, - maxiter=max_opt_iter, - linesearch=LineSearchBackTracking(), + max_time=max_opt_time, + max_iterations=max_opt_iter, + line_searcher=BackTrackingLineSearcher(), + verbosity=verbosity, ) + + @backend_decorator(self.manifold) + def _cost(u, s, vt): + return self._cost(u, s, vt, residual_global) + + @backend_decorator(self.manifold) + def _egrad(u, s, vt): + return self._egrad(u, s, vt, residual_global) + prb = Problem( manifold=self.manifold, - cost=lambda x: self._cost(x, residual_global), - egrad=lambda z: self._egrad(z, residual_global), - verbosity=verbosity, + cost=_cost, + euclidean_gradient=_egrad, ) - solution = solver.solve(prb, x=self.W) - self.W = [solution[0], solution[1], solution[2]] + solution = solver.run(prb, initial_point=self.W) + self.W = [solution.point[0], solution.point[1], solution.point[2]] - return self._cost(self.W, residual_global) + return solution.cost def reset(self): """Reset the model.""" diff --git a/tests/unit/recommenders/models/test_geoimc.py b/tests/unit/recommenders/models/test_geoimc.py index 0eabc339d..2e47f6964 100644 --- a/tests/unit/recommenders/models/test_geoimc.py +++ b/tests/unit/recommenders/models/test_geoimc.py @@ -17,7 +17,7 @@ reduce_dims, ) from pymanopt.manifolds import Stiefel, SymmetricPositiveDefinite -except: +except ImportError: pass # skip if pymanopt not installed @@ -141,9 +141,9 @@ def test_inferer_infer(dataPtr): colFeatureDim = test_data.get_entity("col").shape[1] rank = 2 W = [ - Stiefel(rowFeatureDim, rank).rand(), - SymmetricPositiveDefinite(rank).rand(), - Stiefel(colFeatureDim, rank).rand(), + Stiefel(rowFeatureDim, rank).random_point(), + SymmetricPositiveDefinite(rank).random_point(), + Stiefel(colFeatureDim, rank).random_point(), ] Inferer(method="dot").infer(test_data, W) From cca9669b54bb82c0c35bbfbf47de367ebe763e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Thu, 18 Jul 2024 17:13:54 +0200 Subject: [PATCH 16/29] Use numpy 1.26.4 in Python 3.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fdfa6826f..77817f403 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ "locust>=2.12.2,<3", # requires jinja2 "memory-profiler>=0.61.0,<1", "nltk>=3.8.1,<4", # requires tqdm + "numpy>=1.26.4;python_version>='3.12'", # https://stackoverflow.com/a/77364602/4505998 "notebook>=6.5.5,<8", # requires ipykernel, jinja2, jupyter, nbconvert, nbformat, packaging, requests "numba>=0.57.0,<1", "pandas>2.0.0,<3.0.0", # requires numpy @@ -126,6 +127,6 @@ exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], ), setup_requires=[ - "numpy>=1.26", + "numpy>=1.26.4", ], ) From 5f02f45a2a4d3a6345892ff8009026e19816f232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 31 Jul 2024 11:09:33 +0200 Subject: [PATCH 17/29] Updated pymanopt in aml_utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- tests/ci/azureml_tests/aml_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azureml_tests/aml_utils.py b/tests/ci/azureml_tests/aml_utils.py index d24ec1361..116ee0b7f 100644 --- a/tests/ci/azureml_tests/aml_utils.py +++ b/tests/ci/azureml_tests/aml_utils.py @@ -92,7 +92,7 @@ def get_or_create_environment( - {conda_pkg_jdk} - pip - pip: - - pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip + - pymanopt@git+https://github.com/pymanopt/pymanopt@1de3b6f47258820fdc072fceaeaa763b9fd263b0 - recommenders[dev{",gpu" if use_gpu else ""}{",spark" if use_spark else ""}]@git+https://github.com/recommenders-team/recommenders.git@{commit_sha} """ # See https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04 From 18ff5b7c8e7130ac5967152129a874f310518ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 31 Jul 2024 12:49:27 +0200 Subject: [PATCH 18/29] Relaxed numpy requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a6cf1aa90..e9309a6b2 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "locust>=2.12.2,<3", # requires jinja2 "memory-profiler>=0.61.0,<1", "nltk>=3.8.1,<4", # requires tqdm - "numpy>=1.26.4;python_version>='3.12'", # https://stackoverflow.com/a/77364602/4505998 + "numpy>=1.26.4,<2;python_version>='3.12'", # https://stackoverflow.com/a/77364602/4505998 "notebook>=6.5.5,<8", # requires ipykernel, jinja2, jupyter, nbconvert, nbformat, packaging, requests "numba>=0.57.0,<1", "numpy<2.0.0", # FIXME: Remove numpy<2.0.0 once cornac release a version newer than 2.2.1 that resolve ImportError: numpy.core.multiarray failed to import. @@ -128,6 +128,6 @@ exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], ), setup_requires=[ - "numpy>=1.26.4", + "numpy>=1.15", ], ) From 913abde57e69074b38457cafd05a67b5c6deadab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 31 Jul 2024 15:41:45 +0200 Subject: [PATCH 19/29] Removed Python 3.8 tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- .github/workflows/azureml-cpu-nightly.yml | 2 +- .github/workflows/azureml-gpu-nightly.yml | 2 +- .github/workflows/azureml-spark-nightly.yml | 2 +- .github/workflows/azureml-unit-tests.yml | 2 +- .github/workflows/sarplus.yml | 2 +- recommenders/README.md | 2 +- setup.py | 1 - 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 549926c0d..d3e671d09 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index da5417a40..36f3865bf 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index 6c902fcae..173824b95 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -66,7 +66,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 481d3d80b..3ea14aa95 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -56,7 +56,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/sarplus.yml b/.github/workflows/sarplus.yml index 90d03fef6..5d06f6385 100644 --- a/.github/workflows/sarplus.yml +++ b/.github/workflows/sarplus.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 diff --git a/recommenders/README.md b/recommenders/README.md index 6861a2e6b..89b7142d1 100644 --- a/recommenders/README.md +++ b/recommenders/README.md @@ -14,7 +14,7 @@ Some dependencies require compilation during pip installation. On Linux this can ```bash sudo apt-get install -y build-essential libpython ``` -where `` should be the Python version (e.g. `3.8`). +where `` should be the Python version (e.g. `3.12`). On Windows you will need [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) diff --git a/setup.py b/setup.py index e9309a6b2..25cfecae3 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,6 @@ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 3b0d1a1ec9f72346444743e961053cbba55bea50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 31 Jul 2024 15:49:38 +0200 Subject: [PATCH 20/29] Bumped required python version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 25cfecae3..740606e15 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,7 @@ "numba>=0.57.0,<1", "numpy<2.0.0", # FIXME: Remove numpy<2.0.0 once cornac release a version newer than 2.2.1 that resolve ImportError: numpy.core.multiarray failed to import. "pandas>2.0.0,<3.0.0", # requires numpy - "pandera[strategies]>=0.6.5,<0.18;python_version<='3.8'", # For generating fake datasets - "pandera[strategies]>=0.15.0;python_version>='3.9'", + "pandera[strategies]>=0.15.0", "retrying>=1.3.4,<2", "scikit-learn>=1.2.0,<2", # requires scipy, and introduce breaking change affects feature_extraction.text.TfidfVectorizer.min_df "scikit-surprise>=1.1.3", @@ -121,7 +120,7 @@ "machine learning python spark gpu", install_requires=install_requires, package_dir={"recommenders": "recommenders"}, - python_requires=">=3.6", + python_requires=">=3.9", packages=find_packages( where=".", exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], From 108c8a0924902bc748c5e9c7e2dc8d3ddff56ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Wed, 31 Jul 2024 15:52:19 +0200 Subject: [PATCH 21/29] Upgraded azure python version to python 3.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- .github/workflows/azureml-release-pipeline.yml | 2 +- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/azureml-release-pipeline.yml b/.github/workflows/azureml-release-pipeline.yml index d9899658e..e63bfea4d 100644 --- a/.github/workflows/azureml-release-pipeline.yml +++ b/.github/workflows/azureml-release-pipeline.yml @@ -37,7 +37,7 @@ jobs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Install wheel package run: pip install wheel - name: Create wheel from setup.py diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 4ce6106bf..e6c4dbd31 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -116,7 +116,7 @@ def parse_args(): parser.add_argument( "--python-version", action="store", - default="3.8", + default="3.12", help="Python version", ) parser.add_argument( From 2e2c99af54c85fbd563d80270d3058b3138eb9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Thu, 1 Aug 2024 08:33:08 +0000 Subject: [PATCH 22/29] Updated tf-keras for new python versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- recommenders/models/deeprec/models/dkn.py | 2 ++ setup.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/recommenders/models/deeprec/models/dkn.py b/recommenders/models/deeprec/models/dkn.py index 8b038dd29..a53620313 100644 --- a/recommenders/models/deeprec/models/dkn.py +++ b/recommenders/models/deeprec/models/dkn.py @@ -1,5 +1,7 @@ # Copyright (c) Recommenders contributors. # Licensed under the MIT License. +import os +os.environ['TF_USE_LEGACY_KERAS'] = "1" import numpy as np import tensorflow as tf diff --git a/setup.py b/setup.py index 740606e15..d2a10e5a4 100644 --- a/setup.py +++ b/setup.py @@ -53,9 +53,12 @@ "gpu": [ "fastai>=2.7.11,<3", "nvidia-ml-py>=11.525.84", - "tensorflow>=2.8.4,!=2.9.0.*,!=2.9.1,!=2.9.2,!=2.10.0.*,<2.17", # Fixed TF due to constant security problems and breaking changes #2073 + "tensorflow>=2.8.4,!=2.9.0.*,!=2.9.1,!=2.9.2,!=2.10.0.*,<2.16; python_version<='3.8'", # Fixed TF due to constant security problems and breaking changes #2073 + "tensorflow~=2.16; python_version>'3.8'", # Version needed for python 3.12 "tf-slim>=1.1.0", # No python_requires in its setup.py - "tf-keras<3", + # Use keras 2 instead of keras 3 + "tf-keras~=2.15; python_version<='3.8'", # Not compatible with py3.12 + "tf-keras~=2.16; python_version>'3.8'", "torch>=2.0.1,<3", ], "spark": [ From 8f0265acbb881cc822c3151b749fce99ec531784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Thu, 1 Aug 2024 08:33:46 +0000 Subject: [PATCH 23/29] Revert "Removed Python 3.8 tests" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 913abde57e69074b38457cafd05a67b5c6deadab. Signed-off-by: David Davó --- .github/workflows/azureml-cpu-nightly.yml | 2 +- .github/workflows/azureml-gpu-nightly.yml | 2 +- .github/workflows/azureml-spark-nightly.yml | 2 +- .github/workflows/azureml-unit-tests.yml | 2 +- .github/workflows/sarplus.yml | 2 +- recommenders/README.md | 2 +- setup.py | 1 + 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index d3e671d09..549926c0d 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 36f3865bf..da5417a40 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -67,7 +67,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index 173824b95..6c902fcae 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -66,7 +66,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 3ea14aa95..481d3d80b 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -56,7 +56,7 @@ jobs: strategy: max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} steps: - name: Check out repository code diff --git a/.github/workflows/sarplus.yml b/.github/workflows/sarplus.yml index 5d06f6385..90d03fef6 100644 --- a/.github/workflows/sarplus.yml +++ b/.github/workflows/sarplus.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 diff --git a/recommenders/README.md b/recommenders/README.md index 89b7142d1..6861a2e6b 100644 --- a/recommenders/README.md +++ b/recommenders/README.md @@ -14,7 +14,7 @@ Some dependencies require compilation during pip installation. On Linux this can ```bash sudo apt-get install -y build-essential libpython ``` -where `` should be the Python version (e.g. `3.12`). +where `` should be the Python version (e.g. `3.8`). On Windows you will need [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) diff --git a/setup.py b/setup.py index d2a10e5a4..017f4b23c 100644 --- a/setup.py +++ b/setup.py @@ -112,6 +112,7 @@ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 7705b5d2f3c9dad188ee61dee4a8f3931e77d788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 09:02:15 +0000 Subject: [PATCH 24/29] Revert "Bumped required python version" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3b0d1a1ec9f72346444743e961053cbba55bea50. Signed-off-by: David Davó --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 017f4b23c..e2e00b203 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,8 @@ "numba>=0.57.0,<1", "numpy<2.0.0", # FIXME: Remove numpy<2.0.0 once cornac release a version newer than 2.2.1 that resolve ImportError: numpy.core.multiarray failed to import. "pandas>2.0.0,<3.0.0", # requires numpy - "pandera[strategies]>=0.15.0", + "pandera[strategies]>=0.6.5,<0.18;python_version<='3.8'", # For generating fake datasets + "pandera[strategies]>=0.15.0;python_version>='3.9'", "retrying>=1.3.4,<2", "scikit-learn>=1.2.0,<2", # requires scipy, and introduce breaking change affects feature_extraction.text.TfidfVectorizer.min_df "scikit-surprise>=1.1.3", @@ -124,7 +125,7 @@ "machine learning python spark gpu", install_requires=install_requires, package_dir={"recommenders": "recommenders"}, - python_requires=">=3.9", + python_requires=">=3.6", packages=find_packages( where=".", exclude=["contrib", "docs", "examples", "scenarios", "tests", "tools"], From 4d4779ae43f89f35dad7646faab3cc192e06aadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 11:34:36 +0000 Subject: [PATCH 25/29] #2138 Using requirements-external to specify git deps - Had to do it for python 3.12 and 3.8 compat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- recommenders/README.md | 8 +++++--- requirements-external.txt | 5 +++++ setup.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 requirements-external.txt diff --git a/recommenders/README.md b/recommenders/README.md index 6861a2e6b..9dd38774a 100644 --- a/recommenders/README.md +++ b/recommenders/README.md @@ -74,9 +74,11 @@ We are currently evaluating inclusion of the following dependencies: ## Other dependencies -Some dependencies are not available via the recommenders PyPI package, but can be installed in the following ways: - - pymanopt: this dependency is required for the RLRMC and GeoIMC algorithms; a version of this code compatible with TensorFlow 2 can be - installed with `pip install "git+https://github.com/pymanopt/pymanopt.git@1de3b6f47258820fdc072fceaeaa763b9fd263b0"`. +Some dependencies are not available via the recommenders PyPI package, but can be installed with the [requirements-external.txt](./requirements-external.txt) file: + +``` +pip install -r requirements-external.txt +``` ## NNI dependencies diff --git a/requirements-external.txt b/requirements-external.txt new file mode 100644 index 000000000..c49608b48 --- /dev/null +++ b/requirements-external.txt @@ -0,0 +1,5 @@ +# Jun 2024: Fixes py312 +pymanopt @git+https://github.com/pymanopt/pymanopt@1de3b6f47258820fdc072fceaeaa763b9fd263b0 ; python_version>='3.12' + +# 2024/02/29: pymanopt bumped to python 3.8 +pymanopt @git+https://github.com/pymanopt/pymanopt@e13cecaec3089c790cc93174840b2f557d179b3f ; python_version<'3.12' diff --git a/setup.py b/setup.py index e2e00b203..96717bce7 100644 --- a/setup.py +++ b/setup.py @@ -89,8 +89,8 @@ ] # The following dependency can be installed as below, however PyPI does not allow direct URLs. -# Temporary fix for pymanopt, only this commit works with TF2 -# "git+https://github.com/pymanopt/pymanopt.git@1de3b6f47258820fdc072fceaeaa763b9fd263b0", +# Temporary fix for pymanopt, pypi version does not work with TF2 +# pip install -r requirements-external.txt setup( name="recommenders", From 1cbe6b62a43c79c18d9cd0b37d850118a1900f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 14:22:38 +0000 Subject: [PATCH 26/29] Fixed conda env file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- tests/ci/azureml_tests/aml_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azureml_tests/aml_utils.py b/tests/ci/azureml_tests/aml_utils.py index 116ee0b7f..4433ad9b0 100644 --- a/tests/ci/azureml_tests/aml_utils.py +++ b/tests/ci/azureml_tests/aml_utils.py @@ -92,7 +92,7 @@ def get_or_create_environment( - {conda_pkg_jdk} - pip - pip: - - pymanopt@git+https://github.com/pymanopt/pymanopt@1de3b6f47258820fdc072fceaeaa763b9fd263b0 + - -r requirements-external.txt - recommenders[dev{",gpu" if use_gpu else ""}{",spark" if use_spark else ""}]@git+https://github.com/recommenders-team/recommenders.git@{commit_sha} """ # See https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04 From 734e4ac20226dbfa9f1fab545611416e02e447d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 14:35:59 +0000 Subject: [PATCH 27/29] Changed requirements-external order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- requirements-external.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-external.txt b/requirements-external.txt index c49608b48..07c0edc33 100644 --- a/requirements-external.txt +++ b/requirements-external.txt @@ -1,5 +1,5 @@ -# Jun 2024: Fixes py312 -pymanopt @git+https://github.com/pymanopt/pymanopt@1de3b6f47258820fdc072fceaeaa763b9fd263b0 ; python_version>='3.12' - # 2024/02/29: pymanopt bumped to python 3.8 pymanopt @git+https://github.com/pymanopt/pymanopt@e13cecaec3089c790cc93174840b2f557d179b3f ; python_version<'3.12' + +# Jun 2024: Fixes py312 +pymanopt @git+https://github.com/pymanopt/pymanopt@1de3b6f47258820fdc072fceaeaa763b9fd263b0 ; python_version>='3.12' From 7b01eff1e651ed347300a47370cacf34f4c07a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 14:46:36 +0000 Subject: [PATCH 28/29] Install requirements-external.txt from GitHub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- tests/ci/azureml_tests/aml_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azureml_tests/aml_utils.py b/tests/ci/azureml_tests/aml_utils.py index 4433ad9b0..2afa2c633 100644 --- a/tests/ci/azureml_tests/aml_utils.py +++ b/tests/ci/azureml_tests/aml_utils.py @@ -92,7 +92,7 @@ def get_or_create_environment( - {conda_pkg_jdk} - pip - pip: - - -r requirements-external.txt + - -r https://raw.githubusercontent.com/recommenders-team/recommenders/{commit_sha}/requirements-external.txt - recommenders[dev{",gpu" if use_gpu else ""}{",spark" if use_spark else ""}]@git+https://github.com/recommenders-team/recommenders.git@{commit_sha} """ # See https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04 From 8515dbf17df5dff51c54deb0a1c2d7d6a7baaa41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Dav=C3=B3?= Date: Fri, 2 Aug 2024 15:16:45 +0000 Subject: [PATCH 29/29] Relaxed scipy version for Pyton 3.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Davó --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 96717bce7..ef74ab5a8 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ "retrying>=1.3.4,<2", "scikit-learn>=1.2.0,<2", # requires scipy, and introduce breaking change affects feature_extraction.text.TfidfVectorizer.min_df "scikit-surprise>=1.1.3", - "scipy>=1.10.1,<=1.13.1", # FIXME: Remove scipy<=1.13.1 once cornac release a version newer than 2.2.1. See #2128 + "scipy>=1.9,<=1.13.1", # FIXME: Remove scipy<=1.13.1 once cornac release a version newer than 2.2.1. See #2128 "seaborn>=0.13.0,<1", # requires matplotlib, packaging "transformers>=4.27.0,<5", # requires packaging, pyyaml, requests, tqdm ]