feat: expanded Python docs + Rust docs (#1137)

spiraldb · Oct 29, 2024 · 54c1993 · 54c1993
1 parent e573ab0
commit 54c1993
Show file tree

Hide file tree

Showing 39 changed files with 1,144 additions and 130 deletions.
diff --git a/.github/workflows/python-docs.yml → .github/workflows/docs.yml b/.github/workflows/python-docs.yml → .github/workflows/docs.yml
@@ -1,4 +1,4 @@
-name: Python docs
+name: Python & Rust docs
 
 on:
   push:
@@ -31,6 +31,7 @@ jobs:
 
           built_sha=$(git rev-parse HEAD)
 
+          rm -rf docs/_build/html/rust/CACHETAG.DIR docs/_build/html/rust/debug
           mv docs/_build/html /tmp/html
 
           git fetch origin

diff --git a/docs/Makefile b/docs/Makefile
@@ -18,3 +18,9 @@ help:
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+html: rust-html
+
+.PHONY: rust-html
+rust-html:
+	cargo doc --no-deps --workspace --all-features --target-dir $(BUILDDIR)/html/rust
diff --git a/docs/_static/example.parquet b/docs/_static/example.parquet
diff --git a/docs/_static/example.vortex b/docs/_static/example.vortex
diff --git a/docs/_static/file-format-2024-10-23-1642.svg b/docs/_static/file-format-2024-10-23-1642.svg
diff --git a/docs/_static/style.css b/docs/_static/style.css
@@ -0,0 +1,3 @@
+html .pst-navbar-icon  {
+  font-size: 1.5rem;
+}
diff --git a/docs/_static/vortex_spiral_logo.svg b/docs/_static/vortex_spiral_logo.svg
diff --git a/docs/_static/vortex_spiral_logo_dark_theme.svg b/docs/_static/vortex_spiral_logo_dark_theme.svg
diff --git a/docs/dataset.rst → docs/api/dataset.rst b/docs/dataset.rst → docs/api/dataset.rst
@@ -6,5 +6,15 @@ query engines like DuckDB and Polars. In particular, Vortex will read data propo
 number of rows passing a filter condition and the number of columns in a selection. For most Vortex
 encodings, this property holds true even when the filter condition specifies a single row.
 
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.dataset.VortexDataset
+   ~vortex.dataset.VortexScanner
+
+.. raw:: html
+
+   <hr>
+
 .. automodule:: vortex.dataset
    :members:
diff --git a/docs/api/dtype.rst b/docs/api/dtype.rst
@@ -0,0 +1,27 @@
+Array Data Types
+================
+
+The logical types of the elements of an Array. Each logical type is implemented by a variety of
+Array encodings which describe both a representation-as-bytes as well as how to apply operations on
+that representation.
+
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.dtype.DType
+   ~vortex.dtype.binary
+   ~vortex.dtype.bool
+   ~vortex.dtype.float
+   ~vortex.dtype.int
+   ~vortex.dtype.null
+   ~vortex.dtype.uint
+   ~vortex.dtype.utf8
+
+.. raw:: html
+
+   <hr>
+
+.. automodule:: vortex.dtype
+   :members:
+   :imported-members:
+
diff --git a/docs/api/encoding.rst b/docs/api/encoding.rst
@@ -0,0 +1,26 @@
+Arrays
+======
+
+A Vortex array is a possibly compressed ordered set of homogeneously typed values. Each array has a
+logical type and a physical encoding. The logical type describes the set of operations applicable to
+the values of this array. The physical encoding describes how this array is realized in memory, on
+disk, and over the wire and how to apply operations to that realization.
+
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.encoding.array
+   ~vortex.encoding.compress
+   ~vortex.encoding.Array
+
+.. raw:: html
+
+   <hr>
+
+.. autofunction:: vortex.encoding.array
+
+.. autofunction:: vortex.encoding.compress
+
+.. autoclass:: vortex.encoding.Array
+   :members:
+   :special-members: __len__
diff --git a/docs/api/expr.rst b/docs/api/expr.rst
@@ -0,0 +1,26 @@
+Expressions
+===========
+
+Vortex expressions represent simple filtering conditions on the rows of a Vortex array. For example,
+the following expression represents the set of rows for which the `age` column lies between 23 and
+55:
+
+.. doctest::
+
+   >>> import vortex
+   >>> age = vortex.expr.column("age")
+   >>> (23 > age) & (age < 55)  # doctest: +SKIP
+
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.expr.column
+   ~vortex.expr.Expr
+
+.. raw:: html
+
+   <hr>
+
+.. autofunction:: vortex.expr.column
+
+.. autoclass:: vortex.expr.Expr
diff --git a/docs/api/index.rst b/docs/api/index.rst
@@ -0,0 +1,12 @@
+Python API
+==========
+
+.. toctree::
+   :maxdepth: 5
+
+   encoding
+   dtype
+   io
+   dataset
+   expr
+   scalar
diff --git a/docs/api/io.rst b/docs/api/io.rst
@@ -0,0 +1,20 @@
+Input and Output
+================
+
+Vortex arrays support reading and writing to local and remote file systems, including plain-old
+HTTP, S3, Google Cloud Storage, and Azure Blob Storage.
+
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.io.read_path
+   ~vortex.io.read_url
+   ~vortex.io.write_path
+
+.. raw:: html
+
+   <hr>
+
+.. automodule:: vortex.io
+   :members:
+   :imported-members:
diff --git a/docs/api/scalar.rst b/docs/api/scalar.rst
@@ -0,0 +1,25 @@
+Scalars
+=======
+
+A scalar is a single atomic value like the integer ``1``, the string ``"hello"``, or the structure
+``{"age": 55, "name": "Angela"}``. The :meth:`.Array.scalar_at` method
+returns a native Python value when the cost of doing so is small. However, for larger values like
+binary data, UTF-8 strings, variable-length lists, and structures, Vortex returns a zero-copy *view*
+of the Array data. The ``into_python`` method of each view will copy the scalar into a native Python
+value.
+
+.. autosummary::
+   :nosignatures:
+
+   ~vortex.scalar.Buffer
+   ~vortex.scalar.BufferString
+   ~vortex.scalar.VortexList
+   ~vortex.scalar.VortexStruct
+
+.. raw:: html
+
+   <hr>
+
+.. automodule:: vortex.scalar
+   :members:
+   :imported-members: