ci: update Python versions supported by EVA

.circleci/config.yml

@@ -7,11 +7,9 @@ orbs:
 workflows:
   main:
     jobs:
+        - Pip-PyPI
+        - Pip-Local
         #### UNIT TESTS
-        - Linux:
-            name: "Test | v3.7 | Linux"
-            v: "3.7"
-            mode: "TEST"
         - Linux:
             name: "Test | v3.8 | Linux"
             v: "3.8"
@@ -24,11 +22,13 @@ workflows:
             name: "Test | v3.10 | Linux"
             v: "3.10"
             mode: "TEST"
+        # Ray does not work on 3.11
+        # https://github.com/ray-project/ray/issues/33232
+        # - Linux:
+        #     name: "Test | v3.11 | Linux"
+        #     v: "3.11"
+        #     mode: "TEST"
         ### NOTEBOOKS
-        - Linux:
-            name: "Notebook | v3.7 | Linux"
-            v: "3.7"
-            mode: "NOTEBOOK"
         - Linux:
             name: "Notebook | v3.8 | Linux"
             v: "3.8"
@@ -41,6 +41,10 @@ workflows:
             name: "Notebook | v3.10 | Linux"
             v: "3.10"
             mode: "NOTEBOOK"
+        # - Linux:
+        #     name: "Notebook | v3.11 | Linux"
+        #     v: "3.11"
+        #     mode: "NOTEBOOK"
         ### LINTER
         - Linux:
             name: "Linter | Linux"
@@ -51,7 +55,6 @@ workflows:
             name: "Test | Ray | v3.10 | Linux"
             v: "3.10"
             mode: "RAY"
-        - Pip
         - Windows:
             name: "Windows | v3.10"
         # test_create_index_doesn't work on MacOS
@@ -67,7 +70,7 @@ jobs:
     parameters:
       v:
         type: string
-        default: "3.7"
+        default: "3.10"
       mode:
         type: string
         default: "ALL"
@@ -162,7 +165,7 @@ jobs:
               source test_evadb/bin/activate
               bash script/test/test.sh 
 
-  Pip:
+  Pip-PyPI:
     resource_class: large
     docker:
       - image: "cimg/python:3.10"
@@ -173,4 +176,17 @@ jobs:
           command: |
             pip install --upgrade pip
             pip install evadb
+            bash script/test/package.sh
+
+  Pip-Local:
+    resource_class: large
+    docker:
+      - image: "cimg/python:3.10"
+    steps:
+      - checkout
+      - run:
+          name: Install EVA package locally and start server
+          command: |
+            pip install --upgrade pip
+            pip install "."
             bash script/test/package.sh

.gitignore

@@ -191,3 +191,4 @@ eva.txt
 prof/
 output.txt
 MagicMock/
+queries.txt

eva/server/command_handler.py

@@ -54,8 +54,7 @@ def execute_query_fetch_all(query, **kwargs) -> Optional[Batch]:
         return Batch.concat(batch_list, copy=False)
 
 
-@asyncio.coroutine
-def handle_request(client_writer, request_message):
+async def handle_request(client_writer, request_message):
     """
     Reads a request from a client and processes it
 

script/test/test.sh

@@ -109,7 +109,7 @@ fi
 
 if [[ ( "$OSTYPE" != "msys" ) && ( "$MODE" = "NOTEBOOK" || "$MODE" = "ALL" ) ]];
 then 
-    PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" 
+    PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" --ignore="tutorials/09-license-plate-fuzzy-join.ipynb" --ignore="tutorials/10-toxicity-classifier-huggingface.ipynb"  
     notebook_test_code=$?
     if [ "$notebook_test_code" != "0" ];
     then

setup.py

@@ -14,10 +14,10 @@
 this_directory = Path(__file__).parent
 LONG_DESCRIPTION = (this_directory / "README.md").read_text()
 
-DESCRIPTION = "EVA Video Database System (Think MySQL for videos)."
+DESCRIPTION = "EVA AI-Relational Database System"
 NAME = "evadb"
 AUTHOR = "Georgia Tech Database Group"
-AUTHOR_EMAIL = "[email protected]"
+AUTHOR_EMAIL = "[email protected]"
 URL = "https://github.com/georgia-tech-db/eva"
 
 
@@ -37,9 +37,9 @@ def read(path, encoding="utf-8"):
 VERSION = VERSION_DICT["VERSION"]
 
 minimal_requirement = [
-    "numpy>=1.19.5,<=1.23.5",
-    "opencv-python>=4.5.4.60,<4.6.0.66",  # bug in easyocr
+    "numpy>=1.19.5",
     "pandas>=1.1.5",
+    "opencv-python>=4.6.0.66",
     "Pillow>=8.4.0",
     "sqlalchemy>=1.4.0,<2.0.0",  # major changes in 2.0.0
     "sqlalchemy-utils>=0.36.6",
@@ -49,7 +49,9 @@ def read(path, encoding="utf-8"):
     "ray>=1.13.0",
     "aenum>=2.2.0",
     "diskcache>=5.4.0",
-    "eva-decord==0.6.1",
+    "eva-decord>=0.6.1",
+    "boto3",
+    "nest_asyncio"
 ]
 
 formatter_libs = ["black>=23.1.0", "isort>=5.10.1"]
@@ -94,10 +96,9 @@ def read(path, encoding="utf-8"):
 ### NEEDED FOR A BATTERIES-LOADED EXPERIENCE
 udf_libs = [
     "facenet-pytorch>=2.5.2",  # FACE DETECTION
-    "easyocr>=1.5.0",  # OCR EXTRACTION
     "ipython<8.13.0",  # NOTEBOOKS
     "thefuzz",  # FUZZY STRING MATCHING
-    "ultralytics",  # OBJECT DETECTION (opencv issue due to easy-ocr)
+    "ultralytics>=8.0.93",  # OBJECT DETECTION
     "transformers>=4.27.4",  # HUGGINGFACE
     "openai>=0.27.4",  # CHATGPT
     "timm>=0.6.13",  # HUGGINGFACE VISION TASKS
@@ -133,12 +134,12 @@ def read(path, encoding="utf-8"):
     download_url=DOWNLOAD_URL,
     license=LICENSE,
     classifiers=[
-        "Intended Audience :: Science/Research",
-        "Topic :: Scientific/Engineering :: Information Analysis",
+        "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3",
-        "Development Status :: 3 - Alpha",
-        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        # "Programming Language :: Python :: 3.11",
     ],
     packages=find_packages(exclude=["tests", "tests.*"]),
     # https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html#the-console-scripts-entry-point
@@ -148,7 +149,7 @@ def read(path, encoding="utf-8"):
             "eva_client=eva.eva_cmd_client:main",
         ]
     },
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=INSTALL_REQUIRES,
     extras_require=EXTRA_REQUIRES,
     include_package_data=True,

test/integration_tests/test_error_handling_with_ray.py

@@ -56,17 +56,15 @@ def tearDown(self):
         execute_query_fetch_all(drop_table_query)
 
     def test_ray_error_populate_to_all_stages(self):
-        create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
-                  INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
-                  OUTPUT (labels NDARRAY STR(10),
-                          bboxes NDARRAY FLOAT32(ANYDIM, 4),
-                          scores NDARRAY FLOAT32(ANYDIM))
-                  TYPE  OCRExtraction
-                  IMPL  'eva/udfs/ocr_extractor.py';
+        udf_name, task = "HFObjectDetector", "image-classification"
+        create_udf_query = f"""CREATE UDF {udf_name}
+            TYPE HuggingFace
+            'task' '{task}'
         """
+
         execute_query_fetch_all(create_udf_query)
 
-        select_query = """SELECT OCRExtractor(data) FROM testRayErrorHandling;"""
+        select_query = """SELECT HFObjectDetector(data) FROM testRayErrorHandling;"""
 
         with self.assertRaises(ExecutorError):
             _ = execute_query_fetch_all(select_query)

test/integration_tests/test_like.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import unittest
+from test.markers import ocr_skip_marker
 from test.util import shutdown_ray
 
 from eva.catalog.catalog_manager import CatalogManager
@@ -35,6 +36,7 @@ def tearDown(self):
         # clean up
         execute_query_fetch_all("DROP TABLE IF EXISTS MemeImages;")
 
+    @ocr_skip_marker
     def test_like_with_ocr(self):
         create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
                   INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
@@ -51,6 +53,7 @@ def test_like_with_ocr(self):
         actual_batch = execute_query_fetch_all(select_query)
         self.assertEqual(len(actual_batch), 1)
 
+    @ocr_skip_marker
     def test_like_fails_on_non_string_col(self):
         create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
                   INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))

test/integration_tests/test_pytorch.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 import os
 import unittest
-from test.markers import windows_skip_marker
+from test.markers import ocr_skip_marker, windows_skip_marker
 from test.util import file_remove, load_udfs_for_testing, shutdown_ray
 
 import cv2
@@ -126,6 +126,7 @@ def test_should_run_pytorch_and_facenet(self):
 
     @pytest.mark.torchtest
     @windows_skip_marker
+    @ocr_skip_marker
     def test_should_run_pytorch_and_ocr(self):
         create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
                   INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
@@ -222,6 +223,7 @@ def test_should_run_pytorch_and_similarity(self):
 
     @pytest.mark.torchtest
     @windows_skip_marker
+    @ocr_skip_marker
     def test_should_run_ocr_on_cropped_data(self):
         create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
                   INPUT  (text NDARRAY STR(100))

test/markers.py

@@ -44,3 +44,8 @@
     sys.platform == "linux",
     reason="Test case is duplicate. Disabling to speed up test suite",
 )
+
+ocr_skip_marker = pytest.mark.skipif(
+    sys.platform == "linux",
+    reason="We do not have built-in support for OCR",
+)