From 8ae85b93bc82e9e5df70757db5140ff0c7b81502 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 12:53:48 -0500 Subject: [PATCH 1/9] update ci to attach wasm wheel to Release, cannot upload emscripten wheel to pypi --- .github/workflows/wheels.yml | 46 ++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 5b28e3a..fb193b1 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -99,11 +99,15 @@ jobs: wasm: runs-on: ubuntu-latest + # Pyodide began supporting `micropip.install` from emscripten-compiled binary wheels + # in Pyodide 0.21.0 (Aug 2022), so no need to build wheels for versions before then. + # As of Nov 2022, the matrix for emscripten/python versions since then is simple. + # Update this matrix when new Pyodide versions come out that bump the Python interpreter + # or emscripten version. Ref: https://pyodide.org/en/stable/project/changelog.html strategy: matrix: - # Match Python version, which is 3.10.2 since Pyodide 0.20.0 - # https://pyodide.org/en/stable/project/changelog.html#version-0-20-0 python-version: ["3.10.2"] + emscripten-version: ["3.1.14"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -124,22 +128,20 @@ jobs: - name: Setup emsdk uses: mymindstorm/setup-emsdk@v11 with: - # Match emscripten version, which is 3.1.14 since Pyodide 0.21.0 - # https://pyodide.org/en/stable/project/changelog.html#version-0-21-0 - version: 3.1.14 + version: ${{ matrix.emscripten-version }} - name: Build wheels uses: messense/maturin-action@v1 with: target: wasm32-unknown-emscripten - args: --release --out dist --find-interpreter + args: --release --out wasm_wheel --find-interpreter - name: Upload wheels uses: actions/upload-artifact@v2 with: - name: wheels - path: dist + name: wasm_wheel + path: wasm_wheel - release: - name: Release + pypi-release: + name: Publish to Pypi on Release runs-on: ubuntu-latest needs: - macos @@ -158,3 +160,27 @@ jobs: run: | pip install --upgrade twine twine upload --skip-existing * + + # Can't upload emscripten wheels to Pypi, see https://github.com/pypi/warehouse/issues/10416. + # For now, this will attach the binary wheels to the Release page in Github. Users can + # download those into a pyodide environment and micropip.install from there. + wasm-release: + name: Attach wasm wheel to Release + runs-on: ubuntu-latest + needs: + - wasm + if: startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v3 + with: + name: wasm-wheels + + - name: Attach assets to Release + uses: softprops/action-gh-release@v1 + with: + files: '*.whl' + # 'name' defaults to tag name, as long as release naming is same as tag + # then no need to specify it here. + + + From a72970f510af2bd0bcb8c113076e2ceed5f85b57 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 12:56:38 -0500 Subject: [PATCH 2/9] test action --- .github/workflows/wheels.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fb193b1..b30aeb7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,8 +1,6 @@ name: Build and Release -on: - release: - types: [published] +on: [push] concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 2645640db4d4cf26b9dd69aba7781e9ac6e8bb6f Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 13:11:43 -0500 Subject: [PATCH 3/9] fix artifact name --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b30aeb7..e8947b7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -171,7 +171,7 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: wasm-wheels + name: wasm-wheel - name: Attach assets to Release uses: softprops/action-gh-release@v1 From 39c91cc40135df2c20273fe642301454cedd0501 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 13:21:29 -0500 Subject: [PATCH 4/9] fix artifact name --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e8947b7..7c94658 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -171,7 +171,7 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: wasm-wheel + name: wasm_wheel - name: Attach assets to Release uses: softprops/action-gh-release@v1 From 10693d690a54656dfc3294a6e8f19b1bf9876249 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 17:01:00 -0500 Subject: [PATCH 5/9] update dependencies and tests --- .github/workflows/test.yml | 17 +++++++++-------- README.md | 13 +++++++++++-- pyproject.toml | 2 ++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1deba5..495ee25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,25 +23,26 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v2 + - name: Install Python uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" channels: conda-forge python-version: ${{ matrix.python-version }} + - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: stable profile: minimal default: true - - name: Install Hatch - run: | - pip install hatch + + - name: Install Dependencies + run: pip install pytest maturin + - name: Build Ypy - run: | - unset CONDA_PREFIX - hatch run maturin develop + run: maturin develop + - name: Run Tests - run: | - hatch run pytest + run: pytest diff --git a/README.md b/README.md index a32d053..025161a 100644 --- a/README.md +++ b/README.md @@ -47,17 +47,26 @@ assert value == "hello world!" ## Tests -All tests are located in `/tests`. If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run `pytest` against `py37` through `py311`. To run the tests, install `pytest` and run the command line tool from the project root: +All tests are located in `/tests`. To run the tests, install `pytest` and run the command line tool from the project root: ``` pip install pytest pytest ``` +## Using Hatch + +If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run commands in virtualenvironments for `py37` through `py311`. + +``` +hatch run test:maturin develop +hatch run test:pytest +``` + ## Build Ypy : Build the library as a wheel and store them in `target/wheels`: ``` maturin build -``` +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 373e938..bf9b0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,8 @@ authors = [ readme = "README.md" homepage = "https://github.com/y-crdt/ypy" repository = "https://github.com/y-crdt/ypy" + +[tool.hatch.envs.test] dependencies = ["pytest", "maturin"] [[tool.hatch.envs.test.matrix]] From 38fed46569d036acbeb79d51b96891753e19bfaa Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 17:08:23 -0500 Subject: [PATCH 6/9] run on all push --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 495ee25..94f2257 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,10 @@ name: Tests -on: - push: - branches: [main] - pull_request: - branches: [main] +on: [push] + # push: + # branches: [main] + # pull_request: + # branches: [main] env: CARGO_TERM_COLOR: always @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v2 - + - name: Install Python uses: conda-incubator/setup-miniconda@v2 with: From 0f0e01d27c1d265a35c5dbca1eb17c8da71a19b4 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 18:37:38 -0500 Subject: [PATCH 7/9] add hatch and wasm docs --- README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 025161a..e063c4b 100644 --- a/README.md +++ b/README.md @@ -56,17 +56,72 @@ pytest ## Using Hatch -If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run commands in virtualenvironments for `py37` through `py311`. +If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run commands in virtual environments for `py37` through `py311`. ``` hatch run test:maturin develop hatch run test:pytest ``` -## Build Ypy : +## Build Ypy Build the library as a wheel and store them in `target/wheels`: ``` maturin build +``` + +## Ypy in WASM (Pyodide) + +As a Rust-based library, Ypy cannot build "pure Python" wheels. CI processes build and upload a number of wheels to Pypi, but Pypi does not support hosting `emscripten` / `wasm32` wheels necessary to import in Pyodide (see https://github.com/pypi/warehouse/issues/10416 for more info and updates). For now, Ypy will build `emscripten` wheels and attach the binaries as assets in the appropriate [Releases](https://github.com/y-crdt/ypy/releases) entry. Unfortunately, trying to install directly from the Github download link will result in a CORS error, so you'll need to use a proxy to pull in the binary and write / install from emscripten file system or host the binary somewhere that is CORS accessible for your application. + +You can try out Ypy in Pyodide using the [terminal emulator at pyodide.org](https://pyodide.org/en/stable/console.html): + +``` +Welcome to the Pyodide terminal emulator 🐍 +Python 3.10.2 (main, Sep 15 2022 23:28:12) on WebAssembly/Emscripten +Type "help", "copyright", "credits" or "license" for more information. +>>> wheel_url = 'https://github.com/y-crdt/ypy/releases/download/v0.5.5/y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl' +>>> wheel_name = wheel_url.split('/')[-1] +>>> wheel_name +'y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl' +>>> +>>> proxy_url = f'https://api.allorigins.win/raw?url={wheel_url}' +>>> proxy_url +'https://api.allorigins.win/raw?url=https://github.com/y-crdt/ypy/releases/download/v0.5.5/y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl' +>>> +>>> import pyodide +>>> resp = await pyodide.http.pyfetch(proxy_url) +>>> resp.status +200 +>>> +>>> content = await resp.bytes() +>>> len(content) +360133 +>>> content[:50] +b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xae\xb2}U\x92l\xa7E\xe6\x04\x00\x00u\t\x00\x00\x1d\x00\x00\x00y_py-0.5.5.dist-info' +>>> +>>> with open(wheel_name, 'wb') as f: +... f.write(content) +... +360133 +>>> +>>> import micropip +>>> await micropip.install(f'emfs:./{wheel_name}') +>>> +>>> import y_py as Y +>>> Y + +>>> +>>> d1 = Y.YDoc() +>>> text = d1.get_text('test') +>>> with d1.begin_transaction() as txn: + text.extend(txn, "hello world!") +... +>>> d2 = Y.YDoc() +>>> state_vector = Y.encode_state_vector(d2) +>>> diff = Y.encode_state_as_update(d1, state_vector) +>>> Y.apply_update(d2, diff) +>>> d2.get_text('test') +YText(hello world!) ``` \ No newline at end of file From b45b84221eec3ec8f2f0ff06ba2c057125d8d1f4 Mon Sep 17 00:00:00 2001 From: Kafonek Date: Tue, 29 Nov 2022 18:38:15 -0500 Subject: [PATCH 8/9] revert on release parameters post testing --- .github/workflows/test.yml | 10 +++++----- .github/workflows/wheels.yml | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94f2257..3f878df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,10 @@ name: Tests -on: [push] - # push: - # branches: [main] - # pull_request: - # branches: [main] +on: + push: + branches: [main] + pull_request: + branches: [main] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7c94658..f9069fb 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,6 +1,8 @@ name: Build and Release -on: [push] +on: + release: + types: [published] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -177,8 +179,8 @@ jobs: uses: softprops/action-gh-release@v1 with: files: '*.whl' - # 'name' defaults to tag name, as long as release naming is same as tag - # then no need to specify it here. + # 'name' (release name) defaults to the tag ref in this action. + # If release naming pattern changes, will need to configure 'name' here. From b45988ee0ddd6e33c8657d7c0359c6d3e9779823 Mon Sep 17 00:00:00 2001 From: Matt Kafonek Date: Fri, 9 Dec 2022 14:24:48 -0500 Subject: [PATCH 9/9] Update README.md Co-authored-by: John Waidhofer <33008436+Waidhoferj@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e063c4b..d5a8fdc 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ maturin build ## Ypy in WASM (Pyodide) -As a Rust-based library, Ypy cannot build "pure Python" wheels. CI processes build and upload a number of wheels to Pypi, but Pypi does not support hosting `emscripten` / `wasm32` wheels necessary to import in Pyodide (see https://github.com/pypi/warehouse/issues/10416 for more info and updates). For now, Ypy will build `emscripten` wheels and attach the binaries as assets in the appropriate [Releases](https://github.com/y-crdt/ypy/releases) entry. Unfortunately, trying to install directly from the Github download link will result in a CORS error, so you'll need to use a proxy to pull in the binary and write / install from emscripten file system or host the binary somewhere that is CORS accessible for your application. +As a Rust-based library, Ypy cannot build "pure Python" wheels. CI processes build and upload a number of wheels to PyPI, but PyPI does not support hosting `emscripten` / `wasm32` wheels necessary to import in Pyodide (see https://github.com/pypi/warehouse/issues/10416 for more info and updates). For now, Ypy will build `emscripten` wheels and attach the binaries as assets in the appropriate [Releases](https://github.com/y-crdt/ypy/releases) entry. Unfortunately, trying to install directly from the Github download link will result in a CORS error, so you'll need to use a proxy to pull in the binary and write / install from emscripten file system or host the binary somewhere that is CORS accessible for your application. You can try out Ypy in Pyodide using the [terminal emulator at pyodide.org](https://pyodide.org/en/stable/console.html):