diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5c572199d..840a4427d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,14 +18,14 @@ jobs: fail-fast: false matrix: group: - - Lux # Core Framework - - Boltz # Prebuilt Models using Lux - - LuxLib # Backend of Lux - - Flux2Lux # Flux2Lux Converter + - Lux # Core Framework + - Boltz # Prebuilt Models using Lux + - LuxLib # Backend of Lux + - Flux2Lux # Flux2Lux Converter + - LuxCore # Avoid the heavy Lux Dependency version: - - '1.6' # JET tests are disabled on 1.6 - - '1.7' - - '1.8' + - "1.6" + - "1.8" steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 @@ -41,13 +41,16 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - - uses: julia-actions/julia-buildpkg@v1 + - name: Build Package + run: | + julia --project=. -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.instantiate()' - uses: julia-actions/julia-runtest@v1 env: GROUP: ${{ matrix.group }} + OVERRIDE_INTER_DEPENDENCIES: "false" - uses: julia-actions/julia-processcoverage@v1 with: - directories: src,lib/Boltz/src,lib/LuxLib/src,lib/Flux2Lux/src + directories: src,lib/Boltz/src,lib/LuxLib/src,lib/Flux2Lux/src,lib/LuxCore/src - uses: codecov/codecov-action@v3 with: files: lcov.info diff --git a/.github/workflows/CINightly.yml b/.github/workflows/CINightly.yml index 773faaf4c..dd83a9df8 100644 --- a/.github/workflows/CINightly.yml +++ b/.github/workflows/CINightly.yml @@ -22,6 +22,7 @@ jobs: - Boltz # Prebuilt Models using Lux - LuxLib # Backend of Lux - Flux2Lux # Flux2Lux Converter + - LuxCore # Avoid the heavy Lux Dependency version: - 'nightly' # merge even if tests fail steps: @@ -39,13 +40,16 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - - uses: julia-actions/julia-buildpkg@v1 + - name: Build Package + run: | + julia --project=. -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.instantiate()' - uses: julia-actions/julia-runtest@v1 env: GROUP: ${{ matrix.group }} + OVERRIDE_INTER_DEPENDENCIES: "false" - uses: julia-actions/julia-processcoverage@v1 with: - directories: src,lib/Boltz/src,lib/LuxLib/src,lib/Flux2Lux/src + directories: src,lib/Boltz/src,lib/LuxLib/src,lib/Flux2Lux/src,lib/LuxCore/src - uses: codecov/codecov-action@v3 with: files: lcov.info diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 3a1edd8af..9c711944a 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -27,7 +27,8 @@ jobs: - name: "Run CompatHelper" run: | import CompatHelper - CompatHelper.main(; subdirs=["", "examples", "lib/Boltz", "lib/LuxLib", "lib/Flux2Lux"]) + CompatHelper.main(; subdirs=["", "examples", "examples/ImageNet", "lib/Boltz", + "lib/LuxLib", "lib/Flux2Lux"]) shell: julia --color=yes {0} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index cea4ce36d..237bc5088 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -4,7 +4,7 @@ on: push: branches: - main - tags: '*' + tags: "*" pull_request: concurrency: # Skip intermediate builds: always. @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 with: - version: '1.8' + version: "1.8" - uses: actions/cache@v1 env: cache-name: cache-artifacts @@ -30,9 +30,9 @@ jobs: ${{ runner.os }}-test- ${{ runner.os }}- - name: Install documentation dependencies - run: julia --project=docs -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/Flux2Lux"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxLib"))); Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' + run: julia --project=docs -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxLib"))); Pkg.develop(PackageSpec(path=pwd())); Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/Flux2Lux"))); Pkg.instantiate()' - name: Install examples dependencies - run: julia --project=examples -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' + run: julia --project=examples -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml index 769785cae..7c41ddd52 100644 --- a/.github/workflows/Downstream.yml +++ b/.github/workflows/Downstream.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - julia-version: [1.7] + julia-version: [1.8] os: [ubuntu-latest] package: - { user: SciML, repo: DiffEqFlux.jl, group: BasicNeuralDE } @@ -33,7 +33,9 @@ jobs: with: version: ${{ matrix.julia-version }} arch: x64 - - uses: julia-actions/julia-buildpkg@latest + - name: Build Package + run: | + julia --project=. -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.instantiate()' - name: Clone Downstream uses: actions/checkout@v2 with: @@ -57,6 +59,6 @@ jobs: exit(0) # Exit immediately, as a success end - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 with: files: lcov.info \ No newline at end of file diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index 7b7e4866c..01cd00db8 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - julia-version: [1.7] + julia-version: [1.8] julia-arch: [x86] os: [ubuntu-latest] steps: diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml index 4d0004e83..e903d63d3 100644 --- a/.github/workflows/Invalidations.yml +++ b/.github/workflows/Invalidations.yml @@ -20,14 +20,20 @@ jobs: with: version: '1' - uses: actions/checkout@v3 - - uses: julia-actions/julia-buildpkg@v1 + # - uses: julia-actions/julia-buildpkg@v1 + - name: Build Package + run: | + julia --project=. -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.instantiate()' - uses: julia-actions/julia-invalidations@v1 id: invs_pr - uses: actions/checkout@v3 with: ref: ${{ github.event.repository.default_branch }} - - uses: julia-actions/julia-buildpkg@v1 + # - uses: julia-actions/julia-buildpkg@v1 + - name: Build Package + run: | + julia --project=. -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/LuxCore"))); Pkg.instantiate()' - uses: julia-actions/julia-invalidations@v1 id: invs_default diff --git a/Project.toml b/Project.toml index ceae2eb2d..c55af35fb 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +LuxCore = "bb33d45b-7691-41d6-9220-0943567d0623" LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" @@ -31,6 +32,7 @@ ChainRulesCore = "1" ComponentArrays = "0.13" FillArrays = "0.13" Functors = "0.2, 0.3" +LuxCore = "0.1" LuxLib = "0.1.7" NNlib = "0.8" Optimisers = "0.2" diff --git a/docs/Project.toml b/docs/Project.toml index 7367b2e3b..e1b2eb5be 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,6 +3,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433" Flux2Lux = "ab51a4a6-c8c3-4b1f-af31-4b52a21037df" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +LuxCore = "bb33d45b-7691-41d6-9220-0943567d0623" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" diff --git a/docs/make.jl b/docs/make.jl index c49be79ec..1b98acbc9 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,10 +1,20 @@ -using Documenter, DocumenterMarkdown, Flux2Lux, Lux, LuxLib, Pkg +using Documenter, DocumenterMarkdown, Flux2Lux, LuxCore, Lux, LuxLib, Pkg + +function _setup_subdir_pkgs_index_file(subpkg) + src_file = joinpath(dirname(@__DIR__), "lib", subpkg, "README.md") + dst_file = joinpath(dirname(@__DIR__), "docs/src/lib", subpkg, "index.md") + rm(dst_file; force=true) + cp(src_file, dst_file) + return +end + +_setup_subdir_pkgs_index_file.(["Boltz", "LuxLib", "Flux2Lux", "LuxCore"]) deployconfig = Documenter.auto_detect_deploy_system() Documenter.post_status(deployconfig; type="pending", repo="github.com/avik-pal/Lux.jl.git") makedocs(; sitename="Lux", authors="Avik Pal et al.", clean=true, doctest=true, - modules=[Flux2Lux, Lux, LuxLib], + modules=[Flux2Lux, Lux, LuxLib, LuxCore], strict=[ :doctest, :linkcheck, diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b9dc90840..761a1e54f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -123,6 +123,10 @@ nav: - "LuxLib": - "Introduction": "lib/LuxLib/index.md" - "API Reference": "lib/LuxLib/api.md" + - "LuxCore": + - "Introduction": "lib/LuxCore/index.md" + - "API Reference": "lib/LuxCore/api.md" - "Development Documentation": - "Style Guide": "devdocs/style_guide.md" - "Layer Implementation": "devdocs/layer_implementation.md" + - "Sub Packages": "devdocs/subpackages.md" diff --git a/docs/src/api/core.md b/docs/src/api/core.md index 603c4e79f..56fe68917 100644 --- a/docs/src/api/core.md +++ b/docs/src/api/core.md @@ -2,39 +2,7 @@ CurrentModule = Lux ``` -## Abstract Types - -```@docs -Lux.AbstractExplicitLayer -Lux.AbstractExplicitContainerLayer -``` - -## General - -```@docs -Lux.apply -Lux.setup -``` - -## Parameters - -```@docs -Lux.initialparameters -Lux.parameterlength -``` - -## States - -```@docs -Lux.initialstates -Lux.statelength -Lux.testmode -Lux.trainmode -Lux.update_state -``` - -## Index - -```@index -Pages = ["core.md"] -``` +The documentation for this page has been moved to [LuxCore.jl](../lib/LuxCore/api.md). +However, this the functionality has in **no way been deprecated** and there is no plan +to deprecated access to these functionalities in the future. The change was made merely to +allow lighter dependencies for users extending `Lux.jl`. diff --git a/docs/src/devdocs/layer_implementation.md b/docs/src/devdocs/layer_implementation.md index 374e373ba..4100089be 100644 --- a/docs/src/devdocs/layer_implementation.md +++ b/docs/src/devdocs/layer_implementation.md @@ -41,28 +41,3 @@ varying how calls are made at different timesteps. 1. `reset`ing the hidden-state and memory is slightly tricky. 1. One way would be to store a `initial_hidden_state` and `initial_memory` in the state alongside the `hidden_state` and `memory`. - - -### RNN Blocks - -!!! note - This is currently unimplemented - -An example implementation would be - -```julia -struct RNN{R} <: Lux.AbstractExplicitContainerLayer{(:recurrent_cell,)} - recurrent_cell::R -end - -function (l::RNN)(x::AbstractArray{T,3}, ps::NamedTuple, st::NamedTuple) where {T} - x_init, x_rest = Iterators.peel(eachslice(x; dims=2)) - (y, carry), st = l.recurrent_cell(x_init, ps, st) - for x in x_rest - (y, carry), st = l.recurrent_cell((x, carry), ps, st) - end - return y, st -end -``` - -We enforce the inputs to be of the format `in_dims × sequence_length × batch_size`. diff --git a/docs/src/devdocs/style_guide.md b/docs/src/devdocs/style_guide.md index 0f0c70d43..8d07802c4 100644 --- a/docs/src/devdocs/style_guide.md +++ b/docs/src/devdocs/style_guide.md @@ -27,12 +27,6 @@ We do have automatic formatter, which opens PR after fixing common style issues, * No avoiding multiply symbol -- so `2x` is invalid instead do it like other languages `2 * x`. -## Unicode Characters - -* No use of unicode characters is allowed. -* The only exception is when defining DSLs. In this particular case, how to type the unicode - must be properly documented. - ## Testing !!! note diff --git a/docs/src/devdocs/subpackages.md b/docs/src/devdocs/subpackages.md new file mode 100644 index 000000000..24c8c292e --- /dev/null +++ b/docs/src/devdocs/subpackages.md @@ -0,0 +1,59 @@ +# SubPackages + +`Lux.jl` operates somewhat like a monorepo having a weird inter-dependency structure. So +adding new subpackages in `lib` can be somewhat complicated. Here are the guidelines that +need to be followed: + +## Package Structure + + * Each subpackage should be in its own directory in `lib`. + * Don't have a `docs` directory (see the [Documentation Section](#documentation) for + details). + * Add a `LICENSE` file (needed to register the package independently). + +## Workflows + + * All workflows should go in the `.github/workflows` directory (in the project root). + * For `CI.yml` and `CINightly.yml` + - add the project name to `group` matrix. + - Under `directories` for `julia-actions/julia-processcoverage@v1` add + `lib//src`. + * For `CompatHelper.yml` add `lib/` to the list of `subdirs`. + +## Documentation + + * Create a directory for the package: `docs/src/lib/`. + * Optionally, if you want the `index.md` page for your subpackage to be same as the + `README.md` file, add the package name to `_setup_subdir_pkgs_index_file.([...])` in + `docs/make.jl`. + * For generating documentation (say from docstrings) for your package, you need to update + the documentation pipeline: + - In `.github/workflows/Documentation.yml` under `Install documentation dependencies` + install the package using + `Pkg.develop(PackageSpec(path=joinpath(pwd(), "lib/")))`. + - Add a dependency in `docs/Project.toml` (don't add compat entries). + - Add `using ` in `docs/make.jl`. + - Add the package name to `modules` in `docs/make.jl`. + * For every new page that you have added (including `index.md` if using `README.md` file) + update `docs/mkdocs.yml` `nav` field. + +## Testing + + * For testing, always use `Project.toml` in the `lib//test` directory. + * Write the tests as you would for a normal package. + * In `test/runtests.jl` add the package name to `groups` if `GROUP == "All"`. + * Next list any cross-dependency. When CI is run, it uses the local version of the package + instead of the registered version. + - For example, `Lux` depends on `LuxLib` so `"Lux" => [_get_lib_path("LuxLib")]` + - `Boltz` depends on both `Lux` and `LuxLib` (via `Lux`) so + `"Boltz" => [_get_lib_path("LuxLib"), dirname(@__DIR__)]` + - If there are no cross-dependencies, remember to add an empty vector. + +## Registration + +Registration is simply, just run `@JuliaRegistrator register subdir="lib/"` + +## Code Coverage + +If you have performed all the steps correctly the code coverage for the subpackage will +be available under the flag ``. diff --git a/docs/src/lib/Boltz/.gitkeep b/docs/src/lib/Boltz/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docs/src/lib/Boltz/index.md b/docs/src/lib/Boltz/index.md deleted file mode 100644 index 4d652cf74..000000000 --- a/docs/src/lib/Boltz/index.md +++ /dev/null @@ -1,79 +0,0 @@ -# Boltz ⚡ - -[![Join the chat at https://julialang.zulipchat.com #machine-learning](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/machine-learning) -[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) -[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/lib/Boltz) -[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/lib/Boltz) - -[![CI](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml) -[![CI Nightly](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml) -[![codecov](https://codecov.io/gh/avik-pal/Lux.jl/branch/main/graph/badge.svg?flag=Boltz&token=IMqBM1e3hz)](https://codecov.io/gh/avik-pal/Lux.jl) -[![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/Boltz)](https://pkgs.genieframework.com?packages=Boltz) - -[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) -[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) - -Accelerate ⚡ your ML research using pre-built Deep Learning Models with Lux. - -## Installation - -```julia -using Pkg -Pkg.add("Boltz") -``` - -## Getting Started - -```julia -using Boltz, Lux - -model, ps, st = resnet(:resnet18; pretrained=true) -``` - -## Classification Models - -| MODEL NAME | FUNCTION | NAME | PRETRAINED | TOP 1 ACCURACY (%) | TOP 5 ACCURACY (%) | -| - | - | - | :-: | :-: | :-: | -| AlexNet | `alexnet` | `:alexnet` | ✅ | 54.48 | 77.72 | -| ResNet | `resnet` | `:resnet18` | ✅ | 68.08 | 88.44 | -| ResNet | `resnet` | `:resnet34` | ✅ | 72.13 | 90.91 | -| ResNet | `resnet` | `:resnet50` | ✅ | 74.55 | 92.36 | -| ResNet | `resnet` | `:resnet101` | ✅ | 74.81 | 92.36 | -| ResNet | `resnet` | `:resnet152` | ✅ | 77.63 | 93.84 | -| VGG | `vgg` | `:vgg11` | ✅ | 67.35 | 87.91 | -| VGG | `vgg` | `:vgg13` | ✅ | 68.40 | 88.48 | -| VGG | `vgg` | `:vgg16` | ✅ | 70.24 | 89.80 | -| VGG | `vgg` | `:vgg19` | ✅ | 71.09 | 90.27 | -| VGG | `vgg` | `:vgg11_bn` | ✅ | 69.09 | 88.94 | -| VGG | `vgg` | `:vgg13_bn` | ✅ | 69.66 | 89.49 | -| VGG | `vgg` | `:vgg16_bn` | ✅ | 72.11 | 91.02 | -| VGG | `vgg` | `:vgg19_bn` | ✅ | 72.95 | 91.32 | -| ConvMixer | `convmixer` | `:small` | 🚫 | | | -| ConvMixer | `convmixer` | `:base` | 🚫 | | | -| ConvMixer | `convmixer` | `:large` | 🚫 | | | -| DenseNet | `densenet` | `:densenet121` | 🚫 | | | -| DenseNet | `densenet` | `:densenet161` | 🚫 | | | -| DenseNet | `densenet` | `:densenet169` | 🚫 | | | -| DenseNet | `densenet` | `:densenet201` | 🚫 | | | -| GoogleNet | `googlenet` | `:googlenet` | 🚫 | | | -| MobileNet | `mobilenet` | `:mobilenet_v1` | 🚫 | | | -| MobileNet | `mobilenet` | `:mobilenet_v2` | 🚫 | | | -| MobileNet | `mobilenet` | `:mobilenet_v3_small` | 🚫 | | | -| MobileNet | `mobilenet` | `:mobilenet_v3_large` | 🚫 | | | -| ResNeXT | `resnext` | `:resnext50` | 🚫 | | | -| ResNeXT | `resnext` | `:resnext101` | 🚫 | | | -| ResNeXT | `resnext` | `:resnext152` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:tiny` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:small` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:base` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:large` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:huge` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:giant` | 🚫 | | | -| Vision Transformer | `vision_transformer` | `:gigantic` | 🚫 | | | - -These models can be created using `(; pretrained = )`. - -### Preprocessing - -All the pretrained models require that the images be normalized with the parameters -`mean = [0.485f0, 0.456f0, 0.406f0]` and `std = [0.229f0, 0.224f0, 0.225f0]`. diff --git a/docs/src/lib/LuxCore/api.md b/docs/src/lib/LuxCore/api.md new file mode 100644 index 000000000..350814e52 --- /dev/null +++ b/docs/src/lib/LuxCore/api.md @@ -0,0 +1,40 @@ +```@meta +CurrentModule = LuxCore +``` + +## Abstract Types + +```@docs +LuxCore.AbstractExplicitLayer +LuxCore.AbstractExplicitContainerLayer +``` + +## General + +```@docs +LuxCore.apply +LuxCore.setup +``` + +## Parameters + +```@docs +LuxCore.initialparameters +LuxCore.parameterlength +``` + +## States + +```@docs +LuxCore.initialstates +LuxCore.statelength +LuxCore.testmode +LuxCore.trainmode +LuxCore.update_state +``` + +## Index + +```@index +Pages = ["api.md"] +``` diff --git a/docs/src/lib/LuxLib/index.md b/docs/src/lib/LuxLib/index.md deleted file mode 100644 index a1f58e3ef..000000000 --- a/docs/src/lib/LuxLib/index.md +++ /dev/null @@ -1,29 +0,0 @@ -# LuxLib - -[![Join the chat at https://julialang.zulipchat.com #machine-learning](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/machine-learning) -[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) -[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/lib/LuxLib/) -[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/lib/LuxLib/) - -[![CI](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml) -[![CI Nightly](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml) -[![codecov](https://codecov.io/gh/avik-pal/Lux.jl/branch/main/graph/badge.svg?flag=LuxLib&token=IMqBM1e3hz)](https://codecov.io/gh/avik-pal/Lux.jl) -[![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/LuxLib)](https://pkgs.genieframework.com?packages=LuxLib) - -[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) -[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) - - -Backend for [Lux.jl](http://lux.csail.mit.edu/stable). - -## Tutorials - -This is a developer-facing project and most users **should not** depend on it directly. As -such, we don't have tutorials for this package. Instead, we recommend you check out the -[Lux tutorials](http://lux.csail.mit.edu/stable/examples/examples/). - -## What's the distinction from NNlib.jl? - -Think of this package as a temporary location for functionalities that will move into -NNlib.jl. At the moment, this is supposed to be a heavier dependency than NNlib.jl, and -it makes no attempt to separate code across different architectures. diff --git a/docs/src/manual/interface.md b/docs/src/manual/interface.md index db1766bcc..b7d4265b9 100644 --- a/docs/src/manual/interface.md +++ b/docs/src/manual/interface.md @@ -1,5 +1,11 @@ # Lux Interface +!!! tip + + If you just want to define compatibility with Lux without actually using any of the + other functionality provided by Lux (like layers), it is recommended to depend on + `LuxCore.jl` instead of `Lux.jl`. `LuxCore.jl` is a significantly lighter dependency. + First let's set the expectations straight. - Do you **have to** follow the interface? *No*. diff --git a/lib/LuxCore/LICENSE b/lib/LuxCore/LICENSE new file mode 100644 index 000000000..1f70fe758 --- /dev/null +++ b/lib/LuxCore/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Avik Pal and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/LuxCore/Project.toml b/lib/LuxCore/Project.toml new file mode 100644 index 000000000..bf62d9c0a --- /dev/null +++ b/lib/LuxCore/Project.toml @@ -0,0 +1,14 @@ +name = "LuxCore" +uuid = "bb33d45b-7691-41d6-9220-0943567d0623" +authors = ["Avik Pal and contributors"] +version = "0.1.0" + +[deps] +Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46" + +[compat] +Functors = "0.2, 0.3" +Setfield = "0.8, 1" +julia = "1.6" diff --git a/docs/src/lib/Flux2Lux/index.md b/lib/LuxCore/README.md similarity index 66% rename from docs/src/lib/Flux2Lux/index.md rename to lib/LuxCore/README.md index 892ee728c..7878ca92b 100644 --- a/docs/src/lib/Flux2Lux/index.md +++ b/lib/LuxCore/README.md @@ -1,22 +1,19 @@ -# Flux2Lux +# LuxCore [![Join the chat at https://julialang.zulipchat.com #machine-learning](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/machine-learning) [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) -[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/lib/Flux2Lux/) -[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/lib/Flux2Lux/) +[![Latest Docs](https://img.shields.io/badge/docs-latest-blue.svg)](http://lux.csail.mit.edu/dev/lib/LuxCore/) +[![Stable Docs](https://img.shields.io/badge/docs-stable-blue.svg)](http://lux.csail.mit.edu/stable/lib/LuxCore/) [![CI](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CI.yml) [![CI Nightly](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml/badge.svg)](https://github.com/avik-pal/Lux.jl/actions/workflows/CINightly.yml) -[![codecov](https://codecov.io/gh/avik-pal/Lux.jl/branch/main/graph/badge.svg?flag=Flux2Lux&token=IMqBM1e3hz)](https://codecov.io/gh/avik-pal/Lux.jl) -[![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/Flux2Lux)](https://pkgs.genieframework.com?packages=Flux2Lux) +[![codecov](https://codecov.io/gh/avik-pal/Lux.jl/branch/main/graph/badge.svg?flag=LuxCore&token=IMqBM1e3hz)](https://codecov.io/gh/avik-pal/Lux.jl) +[![Package Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/LuxCore)](https://pkgs.genieframework.com?packages=LuxCore) [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) -Flux2Lux is a package that allows you to convert Flux.jl models to Lux.jl. - -## Difference from `Lux.transform` - -`Lux.transform` has been deprecated in favor of `Flux2Lux.jl`. This package is a strict -superset of its predecessor. It provides additional features like `preserve_ps_st` and -`force_transform`. See the documentation of `Flux2Lux.transform` for more details. +`LuxCore.jl` defines the abstract layers for Lux. Allows users to be compatible with the +entirely of `Lux.jl` without having such a heavy dependency. If you are depending on `Lux.jl` +directly, you do not need to depend on `LuxCore.jl` (all the functionality is exported via +`Lux.jl`). diff --git a/src/core.jl b/lib/LuxCore/src/LuxCore.jl similarity index 93% rename from src/core.jl rename to lib/LuxCore/src/LuxCore.jl index 45c4317ef..eafda05e3 100644 --- a/src/core.jl +++ b/lib/LuxCore/src/LuxCore.jl @@ -1,3 +1,15 @@ +module LuxCore + +using Functors, Random, Setfield + +function _default_rng() + @static if VERSION >= v"1.7" + return Xoshiro(1234) + else + return MersenneTwister(1234) + end +end + """ AbstractExplicitLayer @@ -46,7 +58,7 @@ initialstates(rng::AbstractRNG, l::NamedTuple) = map(Base.Fix1(initialstates, rn Return the total number of parameters of the layer `l`. """ function parameterlength(l::AbstractExplicitLayer) - return parameterlength(initialparameters(Random.default_rng(), l)) + return parameterlength(initialparameters(_default_rng(), l)) end function parameterlength(nt::Union{NamedTuple, Tuple}) return length(nt) == 0 ? 0 : sum(parameterlength, nt) @@ -58,10 +70,10 @@ parameterlength(a::AbstractArray) = length(a) Return the total number of states of the layer `l`. """ -statelength(l::AbstractExplicitLayer) = statelength(initialstates(Random.default_rng(), l)) +statelength(l::AbstractExplicitLayer) = statelength(initialstates(_default_rng(), l)) statelength(nt::Union{NamedTuple, Tuple}) = length(nt) == 0 ? 0 : sum(statelength, nt) statelength(a::AbstractArray) = length(a) -statelength(x::Union{Number, Symbol, Val}) = 1 +statelength(x::Union{Number, Symbol, Val, <:AbstractRNG}) = 1 """ setup(rng::AbstractRNG, l::AbstractExplicitLayer) @@ -87,7 +99,7 @@ function apply(model::AbstractExplicitLayer, x, ps, st::NamedTuple) end function Base.show(io::IO, x::AbstractExplicitLayer) - __t = rsplit(string(get_typename(x)), "."; limit=2) + __t = rsplit(string(Base.typename(typeof(x)).wrapper), "."; limit=2) T = length(__t) == 2 ? __t[2] : __t[1] return print(io, "$T()") end @@ -176,3 +188,5 @@ function _default_layer_check(key) _default_layer_check_closure(x) = hasmethod(keys, (typeof(x),)) ? key ∈ keys(x) : false return _default_layer_check_closure end + +end diff --git a/lib/LuxCore/test/Project.toml b/lib/LuxCore/test/Project.toml new file mode 100644 index 000000000..b7adf6b5d --- /dev/null +++ b/lib/LuxCore/test/Project.toml @@ -0,0 +1,6 @@ +[deps] +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +julia = "1.6" diff --git a/lib/LuxCore/test/runtests.jl b/lib/LuxCore/test/runtests.jl new file mode 100644 index 000000000..08b0f7543 --- /dev/null +++ b/lib/LuxCore/test/runtests.jl @@ -0,0 +1,55 @@ +using LuxCore, Random, Test + +@testset "LuxCore.jl" begin + rng = LuxCore._default_rng() + + @testset "AbstractExplicitLayer Interface" begin + struct Dense <: LuxCore.AbstractExplicitLayer + in::Int + out::Int + end + + function LuxCore.initialparameters(rng::AbstractRNG, l::Dense) + return (w=randn(rng, l.out, l.in), b=randn(rng, l.out)) + end + + model = Dense(5, 6) + ps, st = LuxCore.setup(rng, model) + + @test LuxCore.parameterlength(ps) == LuxCore.parameterlength(model) + @test LuxCore.parameterlength(zeros(10, 2)) == 20 + @test LuxCore.statelength(st) == LuxCore.statelength(model) + @test LuxCore.statelength(zeros(10, 2)) == 20 + @test LuxCore.statelength(Val(true)) == 1 + @test LuxCore.statelength((zeros(10), zeros(5, 2))) == 20 + @test LuxCore.statelength((layer_1=zeros(10), layer_2=zeros(5, 2))) == 20 + end + + @testset "update_state" begin + st = (layer_1=(training=Val(true), val=1), + layer_2=(layer_1=(val=2,), layer_2=(training=Val(true),))) + + st_ = LuxCore.testmode(st) + + @test st_.layer_1.training == Val(false) && + st_.layer_2.layer_2.training == Val(false) && + st_.layer_1.val == st.layer_1.val && + st_.layer_2.layer_1.val == st.layer_2.layer_1.val + + st = st_ + st_ = LuxCore.trainmode(st) + + @test st_.layer_1.training == Val(true) && + st_.layer_2.layer_2.training == Val(true) && + st_.layer_1.val == st.layer_1.val && + st_.layer_2.layer_1.val == st.layer_2.layer_1.val + + st_ = LuxCore.update_state(st, :val, -1) + @test st_.layer_1.training == st.layer_1.training && + st_.layer_2.layer_2.training == st.layer_2.layer_2.training && + st_.layer_1.val == -1 && + st_.layer_2.layer_1.val == -1 + end + + # NOTE(@avik-pal): Custom Layers and Functors are tested in test/core.jl (in Lux) +end diff --git a/src/Lux.jl b/src/Lux.jl index 01765653c..388db9d46 100644 --- a/src/Lux.jl +++ b/src/Lux.jl @@ -22,12 +22,16 @@ using Markdown # Optimisers + ComponentArrays using Optimisers +# LuxCore +using LuxCore +import LuxCore: AbstractExplicitLayer, AbstractExplicitContainerLayer, initialparameters, + initialstates, parameterlength, statelength, update_state, trainmode, + testmode, setup, apply + const use_cuda = Ref{Union{Nothing, Bool}}(nothing) # Utilities include("utils.jl") -# Core -include("core.jl") # Data Transfer Utilities include("adapt.jl") # Layer Implementations diff --git a/test/core.jl b/test/core.jl index 82e621a96..93ca0ca45 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3,20 +3,7 @@ using Functors, Lux, Random, Test rng = Random.default_rng() Random.seed!(rng, 0) -# NOTE(@avik-pal): We have doctests for testing implementation of custom layers - @testset "AbstractExplicitLayer Interface" begin - model = Dense(5, 6) - ps, st = Lux.setup(rng, model) - - @test Lux.parameterlength(ps) == Lux.parameterlength(model) - @test Lux.parameterlength(zeros(10, 2)) == 20 - @test Lux.statelength(st) == Lux.statelength(model) - @test Lux.statelength(zeros(10, 2)) == 20 - @test Lux.statelength(Val(true)) == 1 - @test Lux.statelength((zeros(10), zeros(5, 2))) == 20 - @test Lux.statelength((layer_1=zeros(10), layer_2=zeros(5, 2))) == 20 - # Deprecated Functionality (Remove in v0.5) @test_deprecated Lux.initialparameters(rng, 10) @test_deprecated Lux.initialstates(rng, 10) @@ -28,27 +15,6 @@ end st = (layer_1=(training=Val(true), val=1), layer_2=(layer_1=(val=2,), layer_2=(training=Val(true),))) - st_ = Lux.testmode(st) - - @test st_.layer_1.training == Val(false) && - st_.layer_2.layer_2.training == Val(false) && - st_.layer_1.val == st.layer_1.val && - st_.layer_2.layer_1.val == st.layer_2.layer_1.val - - st = st_ - st_ = Lux.trainmode(st) - - @test st_.layer_1.training == Val(true) && - st_.layer_2.layer_2.training == Val(true) && - st_.layer_1.val == st.layer_1.val && - st_.layer_2.layer_1.val == st.layer_2.layer_1.val - - st_ = Lux.update_state(st, :val, -1) - @test st_.layer_1.training == st.layer_1.training && - st_.layer_2.layer_2.training == st.layer_2.layer_2.training && - st_.layer_1.val == -1 && - st_.layer_2.layer_1.val == -1 - # Deprecated Functionality (Remove in v0.5) @test_deprecated Lux.trainmode(st, true) @test_deprecated Lux.testmode(st, true) diff --git a/test/runtests.jl b/test/runtests.jl index 6b861dff6..a5e81f3ef 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,20 +10,25 @@ function _dev_pkg(path) end groups = if GROUP == "All" - ["Lux", "Boltz", "LuxLib", "Flux2Lux"] + ["Lux", "Boltz", "LuxLib", "Flux2Lux", "LuxCore"] else [GROUP] end -cross_dependencies = Dict("Lux" => [_get_lib_path("LuxLib")], +cross_dependencies = Dict("Lux" => [_get_lib_path("LuxLib"), _get_lib_path("LuxCore")], "Boltz" => [ _get_lib_path("LuxLib"), + _get_lib_path("LuxCore"), dirname(@__DIR__), _get_lib_path("Flux2Lux"), - ], "LuxLib" => [], - "Flux2Lux" => [_get_lib_path("LuxLib"), dirname(@__DIR__)]) + ], + "Flux2Lux" => [ + _get_lib_path("LuxLib"), + _get_lib_path("LuxCore"), + dirname(@__DIR__), + ], "LuxLib" => [], "LuxCore" => []) -const OVERRIDE_INTER_DEPENDENCIES = get(ENV, "OVERRIDE_INTER_DEPENDENCIES", "false") == +const OVERRIDE_INTER_DEPENDENCIES = get(ENV, "OVERRIDE_INTER_DEPENDENCIES", "true") == "true" @time begin for group in groups