Update upper bound for numpy. (#24725)

apache · Feb 8, 2023 · 69ddf44 · 69ddf44
1 parent 287ddf6
commit 69ddf44
Show file tree

Hide file tree

Showing 9 changed files with 27 additions and 51 deletions.
diff --git a/sdks/python/apache_beam/coders/row_coder_test.py b/sdks/python/apache_beam/coders/row_coder_test.py
@@ -417,7 +417,8 @@ def test_batch_encode_decode(self):
     batch_out = coder_impl.create_OutputStream()
     columnar = {
         field: np.array([getattr(person, field) for person in self.PEOPLE],
-                        ndmin=1)
+                        ndmin=1,
+                        dtype=object)
         for field in Person._fields
     }
     coder.encode_batch_to_stream(columnar, batch_out)

diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference.py
@@ -122,7 +122,16 @@ def __init__(self, engine: trt.ICudaEngine):
     self.outputs = []
     self.gpu_allocations = []
     self.cpu_allocations = []
-    """Setup I/O bindings."""
+
+    # TODO(https://github.com/NVIDIA/TensorRT/issues/2557):
+    # Clean up when fixed upstream.
+    try:
+      _ = np.bool  # type: ignore
+    except AttributeError:
+      # numpy >= 1.24.0
+      np.bool = np.bool_  # type: ignore
+
+    # Setup I/O bindings.
     for i in range(self.engine.num_bindings):
       name = self.engine.get_binding_name(i)
       dtype = self.engine.get_binding_dtype(i)

diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt
@@ -36,4 +36,4 @@ protobuf==3.19.4;python_version=="3.10" and sys_platform=="darwin"
 distlib==0.3.1
 
 # Numpy headers
-numpy>=1.14.3,<1.23.0
+numpy>=1.14.3,<1.25
diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
@@ -91,7 +91,7 @@ mmh3==3.0.0
 mock==2.0.0
 nltk==3.8.1
 nose==1.3.7
-numpy==1.22.4
+numpy==1.24.0
 oauth2client==4.1.3
 oauthlib==3.2.2
 objsize==0.6.1

diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
@@ -92,7 +92,7 @@ mmh3==3.0.0
 mock==2.0.0
 nltk==3.8.1
 nose==1.3.7
-numpy==1.22.4
+numpy==1.24.0
 oauth2client==4.1.3
 oauthlib==3.2.2
 objsize==0.6.1

diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
@@ -92,7 +92,7 @@ mmh3==3.0.0
 mock==2.0.0
 nltk==3.8.1
 nose==1.3.7
-numpy==1.22.4
+numpy==1.24.0
 oauth2client==4.1.3
 oauthlib==3.2.2
 objsize==0.6.1

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
@@ -140,7 +140,7 @@ def cythonize(*args, **kwargs):
 if sys.platform == 'win32' and sys.maxsize <= 2**32:
   pyarrow_dependency = ''
 else:
-  pyarrow_dependency = 'pyarrow>=0.15.1,<10.0.0'
+  pyarrow_dependency = 'pyarrow>=3.0.0,<10.0.0'
 
 # We must generate protos after setup_requires are installed.
 def generate_protos_first():
@@ -246,7 +246,9 @@ def get_portability_package_data():
         'grpcio>=1.33.1,!=1.48.0,<2',
         'hdfs>=2.1.0,<3.0.0',
         'httplib2>=0.8,<0.22.0',
-        'numpy>=1.14.3,<1.23.0',
+        # numpy can have breaking changes in minor versions.
+        # Use a strict upper bound.
+        'numpy>=1.14.3,<1.25.0',   # Update build-requirements.txt as well.
         'objsize>=0.6.1,<0.7.0',
         'pymongo>=3.8.0,<4.0.0',
         'proto-plus>=1.7.1,<2',
@@ -343,15 +345,12 @@ def get_portability_package_data():
             'azure-core >=1.7.0',
             'azure-identity >=1.12.0',
           ],
-        #(TODO): Some tests using Pandas implicitly calls inspect.stack()
-        # with python 3.10 leading to incorrect stacktrace.
-        # This can be removed once dill is updated to version > 0.3.5.1
-        # Issue: https://github.com/apache/beam/issues/23566
+        # Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x.
         # Exclude 1.5.0 and 1.5.1 because of
         # https://github.com/pandas-dev/pandas/issues/45725
           'dataframe': [
-            'pandas>=1.0,<1.6,!=1.5.0,!=1.5.1;python_version<"3.10"',
-            'pandas>=1.4.3,<1.6,!=1.5.0,!=1.5.1;python_version>="3.10"'
+            'pandas<1.4.0;python_version=="3.7"',
+            'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"',
           ],
           'dask': [
             'dask >= 2022.6',

diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
@@ -43,19 +43,7 @@ project.tasks.register("preCommitPyCoverage") {
       dependsOn = ["testPy38CloudCoverage"]
 }
 
-// Create a test task for each major version of pyarrow
-toxTask "testPy38pyarrow-0", "py38-pyarrow-0", "${posargs}"
-test.dependsOn "testPy38pyarrow-0"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-0"
-
-toxTask "testPy38pyarrow-1", "py38-pyarrow-1", "${posargs}"
-test.dependsOn "testPy38pyarrow-1"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-1"
-
-toxTask "testPy38pyarrow-2", "py38-pyarrow-2", "${posargs}"
-test.dependsOn "testPy38pyarrow-2"
-preCommitPyCoverage.dependsOn "testPy38pyarrow-2"
-
+// Create a test task for each supported major version of pyarrow
 toxTask "testPy38pyarrow-3", "py38-pyarrow-3", "${posargs}"
 test.dependsOn "testPy38pyarrow-3"
 preCommitPyCoverage.dependsOn "testPy38pyarrow-3"
@@ -85,18 +73,6 @@ test.dependsOn "testPy38pyarrow-9"
 preCommitPyCoverage.dependsOn "testPy38pyarrow-9"
 
 // Create a test task for each minor version of pandas
-toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}"
-test.dependsOn "testPy38pandas-11"
-preCommitPyCoverage.dependsOn "testPy38pandas-11"
-
-toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}"
-test.dependsOn "testPy38pandas-12"
-preCommitPyCoverage.dependsOn "testPy38pandas-12"
-
-toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}"
-test.dependsOn "testPy38pandas-13"
-preCommitPyCoverage.dependsOn "testPy38pandas-13"
-
 toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}"
 test.dependsOn "testPy38pandas-14"
 preCommitPyCoverage.dependsOn "testPy38pandas-14"

diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
@@ -277,14 +277,8 @@ extras = test
 commands =
   {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
 
-[testenv:py{37,38,39,310}-pyarrow-{0,1,2,3,4,5,6,7,8,9}]
+[testenv:py{37,38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
 deps =
-  0: pyarrow>=0.15.1,<0.18.0
-  1: pyarrow>=1,<2
-  2: pyarrow>=2,<3
-  # ARROW-11450,BEAM-11731
-  # pyarrow <3 doesn't work with 1.20.0, but doesn't restrict the bounds
-  {0,1,2}: numpy<1.20.0
   3: pyarrow>=3,<4
   4: pyarrow>=4,<5
   5: pyarrow>=5,<6
@@ -300,12 +294,9 @@ commands =
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
 
-[testenv:py{37,38,39,310}-pandas-{11,12,13,14,15}]
+[testenv:py{37,38,39,310}-pandas-{14,15}]
 deps =
-  11: pandas>=1.1.0,<1.2.0
-  12: pandas>=1.2.0,<1.3.0
-  13: pandas>=1.3.0,<1.4.0
-  14: pandas>=1.4.0,<1.5.0
+  14: pandas>=1.4.3,<1.5.0
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
 commands =