From 925bc9bf873b476c6d36a7cacca90b892c059821 Mon Sep 17 00:00:00 2001 From: Binh Vu Date: Mon, 27 Jun 2022 00:32:15 -0700 Subject: [PATCH] Fix continuous integration (#32) * fix tests * update ci to run test --- .github/workflows/build.sh | 94 +++++++++++++++++++ .github/workflows/ci.yml | 68 ++++++++++++-- .github/workflows/pydiscovery.py | 60 ++++++++++++ .vscode/launch.json | 15 +-- pyproject.toml | 2 +- python/drepr/__init__.py | 7 +- python/drepr/engine.py | 2 +- python/drepr/models/parse_v2/sm_parser.py | 8 ++ python/drepr/models/sm.py | 3 +- python/drepr/outputs/__init__.py | 5 - python/tests/conftest.py | 2 +- python/tests/drepr/models/__init__.py | 0 python/tests/{drepr => }/engine/__init__.py | 0 python/tests/{drepr => }/engine/conftest.py | 0 .../engine/test_complete_description.py | 0 python/tests/{drepr => models}/__init__.py | 0 python/tests/{drepr => }/models/conftest.py | 0 python/tests/{drepr => }/models/test_drepr.py | 0 python/tests/{drepr => }/outputs/__init__.py | 0 python/tests/{drepr => }/outputs/conftest.py | 15 ++- .../tests/{drepr => }/outputs/test_backend.py | 5 +- .../outputs/test_get_data_as_ndarray.py | 9 +- .../{drepr => }/outputs/test_group_by.py | 5 +- .../{drepr => }/outputs/test_iter_records.py | 0 24 files changed, 261 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/build.sh create mode 100644 .github/workflows/pydiscovery.py delete mode 100644 python/tests/drepr/models/__init__.py rename python/tests/{drepr => }/engine/__init__.py (100%) rename python/tests/{drepr => }/engine/conftest.py (100%) rename python/tests/{drepr => }/engine/test_complete_description.py (100%) rename python/tests/{drepr => models}/__init__.py (100%) rename python/tests/{drepr => }/models/conftest.py (100%) rename python/tests/{drepr => }/models/test_drepr.py (100%) rename python/tests/{drepr => }/outputs/__init__.py (100%) rename python/tests/{drepr => }/outputs/conftest.py (58%) rename python/tests/{drepr => }/outputs/test_backend.py (96%) rename python/tests/{drepr => }/outputs/test_get_data_as_ndarray.py (93%) rename python/tests/{drepr => }/outputs/test_group_by.py (91%) rename python/tests/{drepr => }/outputs/test_iter_records.py (100%) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh new file mode 100644 index 0000000..5151f8b --- /dev/null +++ b/.github/workflows/build.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -e + +# Description: builds Python's wheels. +# The script needs yum or apt +# +# Envionment Arguments: (handled by `args.py`) +# PYTHON_HOME: the path to the Python installation, which will be used to build the wheels for. +# Empty if you want to use multiple Pythons by providing PYTHON_HOMES. This has the highest priority. If set, we won't consider PYTHON_HOMES and PYTHON_VERSIONS +# PYTHON_HOMES: comma-separated directories that either contains Python installations or are Python installations. +# PYTHON_VERSIONS: versions of Python separated by comma if you want to restricted to specific versions. +# Arguments: +# -t : target platform. See https://doc.rust-lang.org/nightly/rustc/platform-support.html + +export PATH=$EXTRA_PATH:$PATH + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +while getopts t: flag +do + case "${flag}" in + t) target=${OPTARG};; + esac +done + +if [ -z "$target" ] +then + echo "target is not set (-t ). See more: https://doc.rust-lang.org/nightly/rustc/platform-support.html" + exit 1 +fi + +echo "::group::Setup build tools" +# ############################################## +# to build rocksdb, we need CLang and LLVM +echo "Install CLang and LLVM" +if ! command -v yum &> /dev/null +then + # debian + apt update + apt install -y clang-11 +else + # centos + # https://developers.redhat.com/blog/2018/07/07/yum-install-gcc7-clang# + yum install -y llvm-toolset-7 + source /opt/rh/llvm-toolset-7/enable +fi + +# ############################################## +echo "Install Rust" +if ! command -v cargo &> /dev/null +then + # install rust and cargo + curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + source $HOME/.cargo/env +else + echo "Rust is already installed." + rustup show +fi + +if [ ! -d $(rustc --print target-libdir --target "$target" ) ] +then + rustup target add $target; +fi + +echo "::endgroup::" +echo + +echo "::group::Discovering Python" +IFS=',' read -a PYTHON_HOMES <<< $(MINIMUM_PYTHON_VERSION=3.8 python $SCRIPT_DIR/pydiscovery.py) +if [ ${#PYTHON_HOMES[@]} -eq 0 ]; then + echo "No Python found. Did you forget to set any environment variable PYTHON_HOME or PYTHON_HOMES?" +else + for PYTHON_HOME in "${PYTHON_HOMES[@]}" + do + echo "Found $PYTHON_HOME" + done +fi +echo "::endgroup::" +echo + +for PYTHON_HOME in "${PYTHON_HOMES[@]}" +do + echo "::group::Building for Python $PYTHON_HOME" + + echo "Run: $PYTHON_HOME/bin/pip install maturin" + "$PYTHON_HOME/bin/pip" install maturin + + echo "Run: $PYTHON_HOME/bin/maturin build -r -o dist -i $PYTHON_HOME/bin/python --target $target" + "$PYTHON_HOME/bin/maturin" build -r -o dist -i "$PYTHON_HOME/bin/python" --target $target + + echo "::endgroup::" + echo +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06ccf9..833835f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,37 @@ on: jobs: linux: + strategy: + matrix: + platform: + - target: x86_64-unknown-linux-gnu + image: quay.io/pypa/manylinux2014_x86_64:latest + run_test: true + # - target: i686-unknown-linux-gnu + # image: quay.io/pypa/manylinux2014_i686:latest runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Run test - run: cargo test --no-default-features --features pyo3/auto-initialize - - uses: messense/maturin-action@v1 + - name: Build wheels + run: | + docker run --rm -w /project -v $(pwd):/project \ + -e EXTRA_PATH=/opt/python/cp38-cp38/bin \ + -e PYTHON_HOMES=/opt/python \ + -e CARGO_NET_GIT_FETCH_WITH_CLI=false \ + ${{ matrix.platform.image }} \ + bash /project/.github/workflows/build.sh -t ${{ matrix.platform.target }} + - name: Prepare to run test + if: matrix.platform.run_test == true + uses: actions/setup-python@v4 with: - manylinux: auto - command: build - args: --release -o dist + python-version: 3.8 + - name: Run test + if: matrix.platform.run_test == true + run: | + pip install dist/*cp38*.whl + pip install pytest + mv python/drepr python/drepr2 + pytest -xvs python/tests - name: Upload wheels uses: actions/upload-artifact@v2 with: @@ -23,15 +44,30 @@ jobs: path: dist windows: + strategy: + matrix: + python: ["3.8", "3.9", "3.10"] runs-on: windows-latest steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} - name: Run test run: cargo test --no-default-features --features pyo3/auto-initialize - uses: messense/maturin-action@v1 with: command: build - args: --release --no-sdist -o dist + args: --release --no-sdist -o dist -i python + # - name: Run test + # if: matrix.python == '3.8' + # run: | + # ls dist + # bash -c 'pwd; pip install dist/*cp38*.whl' + # pip install pytest + # mv python/drepr python/drepr2 + # pytest -xvs python/tests - name: Upload wheels uses: actions/upload-artifact@v2 with: @@ -39,15 +75,29 @@ jobs: path: dist macos: + strategy: + matrix: + python: ["3.8", "3.9", "3.10"] runs-on: macos-latest steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} - name: Run test run: cargo test --no-default-features --features pyo3/auto-initialize - uses: messense/maturin-action@v1 with: command: build - args: --release --no-sdist -o dist --universal2 + args: --release --no-sdist -o dist + - name: Run test + if: matrix.python == '3.8' + run: | + pip install dist/*cp38*.whl + pip install pytest + mv python/drepr python/drepr2 + pytest -xvs python/tests - name: Upload wheels uses: actions/upload-artifact@v2 with: @@ -57,7 +107,7 @@ jobs: release: name: Release runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master')" + if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/heads/dev-ci')" needs: [macos, windows, linux] steps: - uses: actions/download-artifact@v2 diff --git a/.github/workflows/pydiscovery.py b/.github/workflows/pydiscovery.py new file mode 100644 index 0000000..ca8fd46 --- /dev/null +++ b/.github/workflows/pydiscovery.py @@ -0,0 +1,60 @@ +from __future__ import print_function +import os, subprocess, re + + +homes = [] +if "PYTHON_HOME" in os.environ: + homes.append(os.environ["PYTHON_HOME"]) +elif "PYTHON_HOMES" in os.environ: + lst = os.environ["PYTHON_HOMES"].split(",") + for path in lst: + if os.path.exists(os.path.join(path, "bin", "python")): + # is the python directory + homes.append(path) + else: + for subpath in os.listdir(path): + if os.path.exists(os.path.join(path, subpath, "bin", "python")): + homes.append(os.path.join(path, subpath)) + +if "PYTHON_VERSIONS" in os.environ: + versions = os.environ["PYTHON_VERSIONS"].split(",") + filtered_homes = [] + for home in homes: + output = ( + subprocess.check_output([os.path.join(home, "bin", "python"), "-V"]) + .decode() + .strip() + ) + for version in versions: + m = re.match("Python ([\d\.)]+)", output) + assert m is not None + pyversion = m.group(1) + if pyversion.startswith(version): + filtered_homes.append(home) + break + + homes = filtered_homes + + +if "MINIMUM_PYTHON_VERSION" in os.environ: + minimum_version = [int(d) for d in os.environ["MINIMUM_PYTHON_VERSION"].split(".")] + filtered_homes = [] + for home in homes: + output = ( + subprocess.check_output([os.path.join(home, "bin", "python"), "-V"]) + .decode() + .strip() + ) + m = re.match(r"Python ([\d\.)]+)", output) + assert m is not None + pyversion = m.group(1).split(".") + + if all( + int(pyversion[i]) >= minimum_version[i] for i in range(len(minimum_version)) + ): + filtered_homes.append(home) + + homes = filtered_homes + +print(",".join(homes)) +exit(0) diff --git a/.vscode/launch.json b/.vscode/launch.json index db54d8f..cb96ead 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,15 +5,16 @@ "version": "0.2.0", "configurations": [ { - "name": "Python: Current File", + "name": "pytest", "type": "python", "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": true, - "env": { - "RUST_BACKTRACE": "1" - } + "module": "pytest", + "args": [ + "-xvs", + "python/tests", + // "python/tests/drepr/outputs/test_get_data_as_ndarray.py::test_get_prop_as_ndarray" + ], + "justMyCode": true } ] } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c22f323..1859706 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "drepr" version = "2.10.0" description = "Data Representation Language for Reading Heterogeneous Datasets" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.8" license = {file = "LICENSE"} authors = [{name = "Binh Vu", email = "binh@toan2.com"}] diff --git a/python/drepr/__init__.py b/python/drepr/__init__.py index 630d818..ef1e590 100644 --- a/python/drepr/__init__.py +++ b/python/drepr/__init__.py @@ -1,6 +1,3 @@ -# from drepr.version import __version__ -# from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat +from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat # from drepr.graph_deprecated import Graph -# from drepr.models import DRepr, DReprBuilder -# from drepr import outputs -# from drepr import models \ No newline at end of file +from drepr.models import DRepr, DReprBuilder \ No newline at end of file diff --git a/python/drepr/engine.py b/python/drepr/engine.py index 2496d20..cc50e41 100644 --- a/python/drepr/engine.py +++ b/python/drepr/engine.py @@ -111,7 +111,7 @@ def execute( if isinstance(output, MemoryOutput) and output.format == OutputFormat.GraphPy: class2nodes = {} for u in ds_model.sm.iter_class_nodes(): - class2nodes[u.node_id] = result["class2nodes"][ + class2nodes[u.node_id] = result[ engine_model.sm_node_idmap[u.node_id] ] return class2nodes diff --git a/python/drepr/models/parse_v2/sm_parser.py b/python/drepr/models/parse_v2/sm_parser.py index 714131b..5757d1e 100644 --- a/python/drepr/models/parse_v2/sm_parser.py +++ b/python/drepr/models/parse_v2/sm_parser.py @@ -119,6 +119,14 @@ def parse(cls, sm: dict) -> SemanticModel: f"{trace1}\nParsing data type") data_type = DataType(data_type) + # normalize value's type (e.g., ruamel.yaml read float into ScalarFloat) + if isinstance(value, str): + value = str(value) + elif isinstance(value, int): + value = int(value) + elif isinstance(value, float): + value = float(value) + node = LiteralNode(node_id=f"lnode:{len(nodes)}", value=value, data_type=data_type) nodes[node.node_id] = node edges[len(edges)] = Edge(len(edges), class_id, node.node_id, predicate) diff --git a/python/drepr/models/sm.py b/python/drepr/models/sm.py index e0d9ca1..628595d 100644 --- a/python/drepr/models/sm.py +++ b/python/drepr/models/sm.py @@ -42,7 +42,8 @@ class DataNode: @dataclass class LiteralNode: node_id: str - value: str + # you should rely on data_type to get the type of value right. The parser may be wrong about it. + value: Union[str, int, float] data_type: Optional[DataType] = None diff --git a/python/drepr/outputs/__init__.py b/python/drepr/outputs/__init__.py index 66d5292..e69de29 100644 --- a/python/drepr/outputs/__init__.py +++ b/python/drepr/outputs/__init__.py @@ -1,5 +0,0 @@ -from .array_backend.array_backend import ArrayBackend -from .graph_backend.graph_backend import GraphBackend -from .namespace import Namespace -from .base_output_sm import FCondition -from .prop_data_ndarray import PropDataNDArray \ No newline at end of file diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 6932813..30a02ef 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -6,7 +6,7 @@ def get_examples_dir(): testdir = Path(os.path.abspath(__file__)).parent - return testdir.parent / "examples" + return testdir.parent.parent / "examples" @pytest.fixture() diff --git a/python/tests/drepr/models/__init__.py b/python/tests/drepr/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/tests/drepr/engine/__init__.py b/python/tests/engine/__init__.py similarity index 100% rename from python/tests/drepr/engine/__init__.py rename to python/tests/engine/__init__.py diff --git a/python/tests/drepr/engine/conftest.py b/python/tests/engine/conftest.py similarity index 100% rename from python/tests/drepr/engine/conftest.py rename to python/tests/engine/conftest.py diff --git a/python/tests/drepr/engine/test_complete_description.py b/python/tests/engine/test_complete_description.py similarity index 100% rename from python/tests/drepr/engine/test_complete_description.py rename to python/tests/engine/test_complete_description.py diff --git a/python/tests/drepr/__init__.py b/python/tests/models/__init__.py similarity index 100% rename from python/tests/drepr/__init__.py rename to python/tests/models/__init__.py diff --git a/python/tests/drepr/models/conftest.py b/python/tests/models/conftest.py similarity index 100% rename from python/tests/drepr/models/conftest.py rename to python/tests/models/conftest.py diff --git a/python/tests/drepr/models/test_drepr.py b/python/tests/models/test_drepr.py similarity index 100% rename from python/tests/drepr/models/test_drepr.py rename to python/tests/models/test_drepr.py diff --git a/python/tests/drepr/outputs/__init__.py b/python/tests/outputs/__init__.py similarity index 100% rename from python/tests/drepr/outputs/__init__.py rename to python/tests/outputs/__init__.py diff --git a/python/tests/drepr/outputs/conftest.py b/python/tests/outputs/conftest.py similarity index 58% rename from python/tests/drepr/outputs/conftest.py rename to python/tests/outputs/conftest.py index 926ee68..9400076 100644 --- a/python/tests/drepr/outputs/conftest.py +++ b/python/tests/outputs/conftest.py @@ -4,15 +4,20 @@ import pytest -from drepr.outputs.array_backend.array_backend import ArrayBackend +try: + from drepr.outputs.array_backend.array_backend import ArrayBackend +except ModuleNotFoundError: + ArrayBackend = None + from drepr.outputs.graph_backend.graph_backend import GraphBackend def get_backends(dataset_dir: Path): - return [ - ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")), - GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")) - ] + backends = [] + if ArrayBackend is not None: + backends.append(ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json"))) + backends.append(GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json"))) + return backends @pytest.fixture() diff --git a/python/tests/drepr/outputs/test_backend.py b/python/tests/outputs/test_backend.py similarity index 96% rename from python/tests/drepr/outputs/test_backend.py rename to python/tests/outputs/test_backend.py index c1e857a..01c7ad9 100644 --- a/python/tests/drepr/outputs/test_backend.py +++ b/python/tests/outputs/test_backend.py @@ -5,7 +5,10 @@ import pytest import orjson -from drepr.outputs.array_backend.array_backend import ArrayBackend +try: + from drepr.outputs.array_backend.array_backend import ArrayBackend +except ModuleNotFoundError: + ArrayBackend = type(None) from drepr.outputs.base_lst_output_class import BaseLstOutputClass from drepr.outputs.base_output_class import BaseOutputClass from drepr.outputs.base_output_sm import BaseOutputSM diff --git a/python/tests/drepr/outputs/test_get_data_as_ndarray.py b/python/tests/outputs/test_get_data_as_ndarray.py similarity index 93% rename from python/tests/drepr/outputs/test_get_data_as_ndarray.py rename to python/tests/outputs/test_get_data_as_ndarray.py index 62b4714..2dabae1 100644 --- a/python/tests/drepr/outputs/test_get_data_as_ndarray.py +++ b/python/tests/outputs/test_get_data_as_ndarray.py @@ -2,7 +2,10 @@ import numpy as np -from drepr.outputs.array_backend.array_backend import ArrayBackend +try: + from drepr.outputs.array_backend.array_backend import ArrayBackend +except ModuleNotFoundError: + ArrayBackend = type(None) from drepr.outputs.base_output_sm import BaseOutputSM from drepr.outputs.namespace import Namespace @@ -73,4 +76,6 @@ def test_get_prop_as_ndarray(s01: List[BaseOutputSM], s02: List[BaseOutputSM], s assert records[i].s(mint_geo.lat) == data.index_props[0][i] assert records[i].s(mint_geo.long) == data.index_props[1][i] - assert len(array_values) == 0 and len(graph_values) == 0 + if ArrayBackend is not type(None): + assert len(array_values) == 0 + assert len(graph_values) == 0 diff --git a/python/tests/drepr/outputs/test_group_by.py b/python/tests/outputs/test_group_by.py similarity index 91% rename from python/tests/drepr/outputs/test_group_by.py rename to python/tests/outputs/test_group_by.py index 951f478..7ce6818 100644 --- a/python/tests/drepr/outputs/test_group_by.py +++ b/python/tests/outputs/test_group_by.py @@ -1,6 +1,9 @@ from typing import List, Dict, Tuple, Callable, Any, Optional -from drepr.outputs.array_backend.array_backend import ArrayBackend +try: + from drepr.outputs.array_backend.array_backend import ArrayBackend +except ModuleNotFoundError: + ArrayBackend = type(None) from drepr.outputs.base_output_sm import BaseOutputSM from drepr.outputs.record_id import GraphRecordID, BlankRecordID diff --git a/python/tests/drepr/outputs/test_iter_records.py b/python/tests/outputs/test_iter_records.py similarity index 100% rename from python/tests/drepr/outputs/test_iter_records.py rename to python/tests/outputs/test_iter_records.py