Fix continuous integration (#32)

* fix tests
* update ci to run test

.github/workflows/build.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+set -e
+
+# Description: builds Python's wheels.
+# The script needs yum or apt
+#
+# Envionment Arguments: (handled by `args.py`)
+#   PYTHON_HOME: the path to the Python installation, which will be used to build the wheels for. 
+#        Empty if you want to use multiple Pythons by providing PYTHON_HOMES. This has the highest priority. If set, we won't consider PYTHON_HOMES and PYTHON_VERSIONS
+#   PYTHON_HOMES: comma-separated directories that either contains Python installations or are Python installations.
+#   PYTHON_VERSIONS: versions of Python separated by comma if you want to restricted to specific versions.
+# Arguments:
+#   -t <target>: target platform. See https://doc.rust-lang.org/nightly/rustc/platform-support.html
+
+export PATH=$EXTRA_PATH:$PATH
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+while getopts t: flag
+do
+    case "${flag}" in
+        t) target=${OPTARG};;
+    esac
+done
+
+if [ -z "$target" ]
+then
+    echo "target is not set (-t <target>). See more: https://doc.rust-lang.org/nightly/rustc/platform-support.html"
+    exit 1
+fi
+
+echo "::group::Setup build tools"
+# ##############################################
+# to build rocksdb, we need CLang and LLVM
+echo "Install CLang and LLVM"
+if ! command -v yum &> /dev/null
+then
+    # debian
+    apt update
+    apt install -y clang-11
+else
+    # centos
+    # https://developers.redhat.com/blog/2018/07/07/yum-install-gcc7-clang#
+    yum install -y llvm-toolset-7
+    source /opt/rh/llvm-toolset-7/enable
+fi
+
+# ##############################################
+echo "Install Rust"
+if ! command -v cargo &> /dev/null
+then
+    # install rust and cargo
+    curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+    source $HOME/.cargo/env
+else
+    echo "Rust is already installed."
+    rustup show
+fi
+
+if [ ! -d $(rustc --print target-libdir --target "$target" ) ]
+then
+    rustup target add $target;
+fi
+
+echo "::endgroup::"
+echo
+
+echo "::group::Discovering Python"
+IFS=',' read -a PYTHON_HOMES <<< $(MINIMUM_PYTHON_VERSION=3.8 python $SCRIPT_DIR/pydiscovery.py)
+if [ ${#PYTHON_HOMES[@]} -eq 0 ]; then
+    echo "No Python found. Did you forget to set any environment variable PYTHON_HOME or PYTHON_HOMES?"
+else
+    for PYTHON_HOME in "${PYTHON_HOMES[@]}"
+    do
+        echo "Found $PYTHON_HOME"
+    done    
+fi
+echo "::endgroup::"
+echo
+
+for PYTHON_HOME in "${PYTHON_HOMES[@]}"
+do
+    echo "::group::Building for Python $PYTHON_HOME"
+
+    echo "Run: $PYTHON_HOME/bin/pip install maturin"    
+    "$PYTHON_HOME/bin/pip" install maturin
+
+    echo "Run: $PYTHON_HOME/bin/maturin build -r -o dist -i $PYTHON_HOME/bin/python --target $target"
+    "$PYTHON_HOME/bin/maturin" build -r -o dist -i "$PYTHON_HOME/bin/python" --target $target
+
+    echo "::endgroup::"
+    echo
+done

.github/workflows/ci.yml

@@ -6,48 +6,98 @@ on:
 
 jobs:
   linux:
+    strategy:
+      matrix:
+        platform:
+          - target: x86_64-unknown-linux-gnu
+            image: quay.io/pypa/manylinux2014_x86_64:latest
+            run_test: true
+          # - target: i686-unknown-linux-gnu
+          #   image: quay.io/pypa/manylinux2014_i686:latest
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Run test
-        run: cargo test --no-default-features --features pyo3/auto-initialize
-      - uses: messense/maturin-action@v1
+      - name: Build wheels
+        run: |
+          docker run --rm -w /project -v $(pwd):/project \
+            -e EXTRA_PATH=/opt/python/cp38-cp38/bin \
+            -e PYTHON_HOMES=/opt/python \
+            -e CARGO_NET_GIT_FETCH_WITH_CLI=false \
+            ${{ matrix.platform.image }} \
+            bash /project/.github/workflows/build.sh -t ${{ matrix.platform.target }}
+      - name: Prepare to run test
+        if: matrix.platform.run_test == true
+        uses: actions/setup-python@v4
         with:
-          manylinux: auto
-          command: build
-          args: --release -o dist
+          python-version: 3.8
+      - name: Run test
+        if: matrix.platform.run_test == true
+        run: |
+          pip install dist/*cp38*.whl
+          pip install pytest
+          mv python/drepr python/drepr2
+          pytest -xvs python/tests
       - name: Upload wheels
         uses: actions/upload-artifact@v2
         with:
           name: wheels
           path: dist
 
   windows:
+    strategy:
+      matrix:
+        python: ["3.8", "3.9", "3.10"]
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
       - name: Run test
         run: cargo test --no-default-features --features pyo3/auto-initialize
       - uses: messense/maturin-action@v1
         with:
           command: build
-          args: --release --no-sdist -o dist
+          args: --release --no-sdist -o dist -i python
+      # - name: Run test
+      #   if: matrix.python == '3.8'
+      #   run: |
+      #     ls dist
+      #     bash -c 'pwd; pip install dist/*cp38*.whl'
+      #     pip install pytest
+      #     mv python/drepr python/drepr2
+      #     pytest -xvs python/tests
       - name: Upload wheels
         uses: actions/upload-artifact@v2
         with:
           name: wheels
           path: dist
 
   macos:
+    strategy:
+      matrix:
+        python: ["3.8", "3.9", "3.10"]
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
       - name: Run test
         run: cargo test --no-default-features --features pyo3/auto-initialize
       - uses: messense/maturin-action@v1
         with:
           command: build
-          args: --release --no-sdist -o dist --universal2
+          args: --release --no-sdist -o dist
+      - name: Run test
+        if: matrix.python == '3.8'
+        run: |
+          pip install dist/*cp38*.whl
+          pip install pytest
+          mv python/drepr python/drepr2
+          pytest -xvs python/tests
       - name: Upload wheels
         uses: actions/upload-artifact@v2
         with:
@@ -57,7 +107,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master')"
+    if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/heads/dev-ci')"
     needs: [macos, windows, linux]
     steps:
       - uses: actions/download-artifact@v2

.github/workflows/pydiscovery.py

@@ -0,0 +1,60 @@
+from __future__ import print_function
+import os, subprocess, re
+
+
+homes = []
+if "PYTHON_HOME" in os.environ:
+    homes.append(os.environ["PYTHON_HOME"])
+elif "PYTHON_HOMES" in os.environ:
+    lst = os.environ["PYTHON_HOMES"].split(",")
+    for path in lst:
+        if os.path.exists(os.path.join(path, "bin", "python")):
+            # is the python directory
+            homes.append(path)
+        else:
+            for subpath in os.listdir(path):
+                if os.path.exists(os.path.join(path, subpath, "bin", "python")):
+                    homes.append(os.path.join(path, subpath))
+
+if "PYTHON_VERSIONS" in os.environ:
+    versions = os.environ["PYTHON_VERSIONS"].split(",")
+    filtered_homes = []
+    for home in homes:
+        output = (
+            subprocess.check_output([os.path.join(home, "bin", "python"), "-V"])
+            .decode()
+            .strip()
+        )
+        for version in versions:
+            m = re.match("Python ([\d\.)]+)", output)
+            assert m is not None
+            pyversion = m.group(1)
+            if pyversion.startswith(version):
+                filtered_homes.append(home)
+                break
+
+    homes = filtered_homes
+
+
+if "MINIMUM_PYTHON_VERSION" in os.environ:
+    minimum_version = [int(d) for d in os.environ["MINIMUM_PYTHON_VERSION"].split(".")]
+    filtered_homes = []
+    for home in homes:
+        output = (
+            subprocess.check_output([os.path.join(home, "bin", "python"), "-V"])
+            .decode()
+            .strip()
+        )
+        m = re.match(r"Python ([\d\.)]+)", output)
+        assert m is not None
+        pyversion = m.group(1).split(".")
+
+        if all(
+            int(pyversion[i]) >= minimum_version[i] for i in range(len(minimum_version))
+        ):
+            filtered_homes.append(home)
+
+    homes = filtered_homes
+
+print(",".join(homes))
+exit(0)

.vscode/launch.json

@@ -5,15 +5,16 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Python: Current File",
+            "name": "pytest",
             "type": "python",
             "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal",
-            "justMyCode": true,
-            "env": {
-                "RUST_BACKTRACE": "1"
-            }
+            "module": "pytest",
+            "args": [
+                "-xvs",
+                "python/tests",
+                // "python/tests/drepr/outputs/test_get_data_as_ndarray.py::test_get_prop_as_ndarray"
+            ],
+            "justMyCode": true
         }
     ]
 }

pyproject.toml

@@ -3,7 +3,7 @@ name = "drepr"
 version = "2.10.0"
 description = "Data Representation Language for Reading Heterogeneous Datasets"
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 license = {file = "LICENSE"}
 authors = [{name = "Binh Vu", email = "[email protected]"}]
 

python/drepr/__init__.py

@@ -1,6 +1,3 @@
-# from drepr.version import __version__
-# from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat
+from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat
 # from drepr.graph_deprecated import Graph
-# from drepr.models import DRepr, DReprBuilder
-# from drepr import outputs
-# from drepr import models
+from drepr.models import DRepr, DReprBuilder

python/drepr/engine.py

@@ -111,7 +111,7 @@ def execute(
     if isinstance(output, MemoryOutput) and output.format == OutputFormat.GraphPy:
         class2nodes = {}
         for u in ds_model.sm.iter_class_nodes():
-            class2nodes[u.node_id] = result["class2nodes"][
+            class2nodes[u.node_id] = result[
                 engine_model.sm_node_idmap[u.node_id]
             ]
         return class2nodes

python/drepr/models/parse_v2/sm_parser.py

@@ -119,6 +119,14 @@ def parse(cls, sm: dict) -> SemanticModel:
                                       f"{trace1}\nParsing data type")
                     data_type = DataType(data_type)
 
+                # normalize value's type (e.g., ruamel.yaml read float into ScalarFloat)
+                if isinstance(value, str):
+                    value = str(value)
+                elif isinstance(value, int):
+                    value = int(value)
+                elif isinstance(value, float):
+                    value = float(value)
+
                 node = LiteralNode(node_id=f"lnode:{len(nodes)}", value=value, data_type=data_type)
                 nodes[node.node_id] = node
                 edges[len(edges)] = Edge(len(edges), class_id, node.node_id, predicate)

python/drepr/models/sm.py

@@ -42,7 +42,8 @@ class DataNode:
 @dataclass
 class LiteralNode:
     node_id: str
-    value: str
+    # you should rely on data_type to get the type of value right. The parser may be wrong about it.
+    value: Union[str, int, float]
     data_type: Optional[DataType] = None
 
 

python/drepr/outputs/__init__.py

@@ -1,5 +0,0 @@
-from .array_backend.array_backend import ArrayBackend
-from .graph_backend.graph_backend import GraphBackend
-from .namespace import Namespace
-from .base_output_sm import FCondition
-from .prop_data_ndarray import PropDataNDArray

python/tests/conftest.py

@@ -6,7 +6,7 @@
 
 def get_examples_dir():
     testdir = Path(os.path.abspath(__file__)).parent
-    return testdir.parent / "examples"
+    return testdir.parent.parent / "examples"
 
 
 @pytest.fixture()

python/tests/drepr/outputs/conftest.py → python/tests/outputs/conftest.py

@@ -4,15 +4,20 @@
 
 import pytest
 
-from drepr.outputs.array_backend.array_backend import ArrayBackend
+try:
+    from drepr.outputs.array_backend.array_backend import ArrayBackend
+except ModuleNotFoundError:
+    ArrayBackend = None
+
 from drepr.outputs.graph_backend.graph_backend import GraphBackend
 
 
 def get_backends(dataset_dir: Path):
-    return [
-        ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")),
-        GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json"))
-    ]
+    backends = []
+    if ArrayBackend is not None:
+        backends.append(ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")))
+    backends.append(GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")))
+    return backends
 
 
 @pytest.fixture()