pythonlib: improved dependency management and examples (com-lihaoyi#3998

)

Make the dependency management more flexible:

- [x] allow users to re-use `requirements.txt` files (inspired by [the
same pants
feature](https://www.pantsbuild.org/dev/docs/python/overview/third-party-dependencies#requirementstxt))
- [x] separate library and tool requirements
- library deps are transitive, and will in the future be included as
requirements in distributions
- tool requirements are not transitive and won't be included in
distributions. They're used for things like type checkers and bundlers
(which can all be extracted into separated traits later on if we want
to).
- [x] support referencing wheel files directly
- [x] allow user-defined PYTHONPATH
- [x] custom package repos
- ~lockfile support~ Removing this from this pull request, since it's
likely a bit more complex than I initially thought. Pants delegates this
behavior to PEX, but we should see if this can be done with more basic
tools, or we decide to rely on pex for everything.
- [x] examples for all the above

Part of com-lihaoyi#3928

docs/modules/ROOT/nav.adoc

@@ -29,16 +29,9 @@
 ** xref:kotlinlib/publishing.adoc[]
 // ** xref:kotlinlib/build-examples.adoc[]
 ** xref:kotlinlib/web-examples.adoc[]
-* xref:pythonlib/intro.adoc[]
 
-// Future Additions
-// ** xref:pythonlib/module-config.adoc[]
-// ** xref:pythonlib/dependencies.adoc[]
-// ** xref:pythonlib/testing.adoc[]
-// ** xref:pythonlib/linting.adoc[]
-// ** xref:pythonlib/publishing.adoc[]
-// ** xref:pythonlib/build-examples.adoc[]
-// ** xref:pythonlib/web-examples.adoc[]
+* xref:pythonlib/intro.adoc[]
+** xref:pythonlib/dependencies.adoc[]
 
 * (Experimental) Android with Mill
 ** xref:android/java.adoc[]

docs/modules/ROOT/pages/pythonlib/dependencies.adoc

@@ -0,0 +1,31 @@
+= Python Library Dependencies
+
+include::partial$gtag-config.adoc[]
+
+
+This page goes into more detail about configuring third party dependencies
+for `PythonModule`s.
+
+== Adding Dependencies
+
+include::partial$example/pythonlib/dependencies/1-pip-deps.adoc[]
+
+=== Adding Dependencies via requirements.txt files
+
+include::partial$example/pythonlib/dependencies/2-pip-requirements.adoc[]
+
+== Unmanaged Wheels
+
+include::partial$example/pythonlib/dependencies/3-unmanaged-wheels.adoc[]
+
+== Downloading Unmanaged Wheels
+
+include::partial$example/pythonlib/dependencies/4-downloading-unmanaged-wheels.adoc[]
+
+== Using Custom Package Indexes
+
+include::partial$example/pythonlib/dependencies/5-repository-config.adoc[]
+
+== Debugging
+
+include::partial$example/pythonlib/dependencies/6-debugging.adoc[]

docs/modules/ROOT/pages/pythonlib/intro.adoc

@@ -19,4 +19,4 @@ include::partial$example/pythonlib/basic/2-custom-build-logic.adoc[]
 
 == Multi-Module Project
 
-include::partial$example/pythonlib/basic/3-multi-module.adoc[]
+include::partial$example/pythonlib/basic/3-multi-module.adoc[]

example/package.mill

@@ -65,7 +65,8 @@ object `package` extends RootModule with Module {
     object basic extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "basic"))
   }
   object pythonlib extends Module {
-    object basic extends Cross[ExampleCrossModulePython](build.listIn(millSourcePath / "basic"))
+    object basic extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "basic"))
+    object dependencies extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "dependencies"))
   }
 
   object cli extends Module{
@@ -99,40 +100,6 @@ object `package` extends RootModule with Module {
     object typescript extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "typescript"))
   }
 
-  trait ExampleCrossModulePython extends ExampleCrossModuleJava {
-    override def lineTransform(line: String) ={
-      this.millModuleSegments.parts.last match {
-        case "1-simple" =>
-          val updatedLine = line
-          .replace("xref:{language-small}lib/web-examples.adoc", "link:") // Need updated link
-          .replace("xref:{language-small}lib/build-examples.adoc", "link:") // Need updated link
-          .replace("compile", "typeCheck")
-          .replace("Scala console", "Python console")
-          .replace("Ammonite Scala", "Python")
-          .replace("assembly", "typeCheck")
-          .replace(s"// $$ mill jar           # bundle the classfiles into a jar suitable for publishing", "")
-          .replace("foo.scalaVersion", "foo.typeCheck")
-          updatedLine
-        case "2-custom-build-logic" =>
-          val updatedLine = line
-          .replace("17", "10") // it's just the change for page count
-          .replace("`allSourceFiles` (an existing task)", "`allSourceFiles`")
-          updatedLine
-
-        case "3-multi-module" =>
-          val updatedLine = line
-          .replace("compiled", "typeChecked")
-          .replace("compile", "typeCheck")
-          .replace("...bar.BarTests...simple...", "test_escaping (...test.TestScript...) ... ok")
-          .replace("...bar.BarTests...escaping...", "test_simple (...test.TestScript...) ... ok")
-          updatedLine
-
-        case _ => line
-
-      }
-    }
-  }
-
   trait ExampleCrossModuleKotlin extends ExampleCrossModuleJava {
 
     override def lineTransform(line: String) = this.millModuleSegments.parts.last match {

example/pythonlib/basic/1-simple/build.mill

@@ -1,20 +1,18 @@
-//// SNIPPET:BUILD
 package build
 import mill._, pythonlib._
 
 object foo extends PythonModule {
 
   def mainScript = Task.Source { millSourcePath / "src" / "foo.py" }
 
-  def pythonDeps = Agg("Jinja2==3.1.4")
+  def pythonDeps = Seq("Jinja2==3.1.4")
 
   object test extends PythonTests with TestModule.Unittest
 
 }
 
 // This is a basic Mill build for a single `PythonModule`, with one
 // dependency and a test suite using the `Unittest` Library.
-//// SNIPPET:TREE
 //
 // ----
 // build.mill
@@ -35,7 +33,6 @@ object foo extends PythonModule {
 //         run.dest/
 //         ...
 // ----
-//// SNIPPET:DEPENDENCIES
 //
 // This example project uses one dependency - https://pypi.org/project/Jinja2/[Jinja2]
 // for HTML rendering and uses it to wrap a given input string in HTML templates with proper escaping.

example/pythonlib/basic/2-custom-build-logic/build.mill

@@ -1,4 +1,3 @@
-//// SNIPPET:BUILD
 package build
 import mill._, pythonlib._
 

example/pythonlib/basic/3-multi-module/build.mill

@@ -1,4 +1,3 @@
-//// SNIPPET:BUILD
 package build
 import mill._, pythonlib._
 
@@ -13,10 +12,9 @@ object foo extends MyModule {
 
 object bar extends MyModule {
   def mainScript = Task.Source { millSourcePath / "src" / "bar.py" }
-  def pythonDeps = Agg("Jinja2==3.1.4")
+  def pythonDeps = Seq("Jinja2==3.1.4")
 }
 //
-//// SNIPPET:TREE
 // ----
 // build.mill
 // foo/

example/pythonlib/dependencies/1-pip-deps/build.mill

@@ -0,0 +1,22 @@
+package build
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule {
+  def pythonDeps = Seq(
+    "numpy==2.1.2",
+    "pandas~=2.2.3",
+    "jinja2 @ https://github.com/pallets/jinja/releases/download/3.1.4/jinja2-3.1.4-py3-none-any.whl"
+  )
+}
+
+// You can define the `pythonDeps` field to add dependencies to your module, which will be installed
+// via https://pip.pypa.io/en/stable/[pip]. Dependencies can include
+// https://peps.python.org/pep-0440/[anything that pip understands], such as `<package>==<version>`
+// constraints, or even direct references to wheels.
+
+/** Usage
+
+> ./mill run
+[10 20 30 40 50]
+
+*/

example/pythonlib/dependencies/1-pip-deps/src/main.py

@@ -0,0 +1,4 @@
+import numpy as np
+
+data = np.array([10, 20, 30, 40, 50])
+print(data)

example/pythonlib/dependencies/2-pip-requirements/build.mill

@@ -0,0 +1,18 @@
+// You can also read dependencies from `requirements.txt` files. This can be
+// useful if you're migrating an existing project to mill.
+
+package build
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule {
+  def pythonRequirementFiles = Task.Sources {
+    millSourcePath / "requirements.txt"
+  }
+}
+
+/** Usage
+
+> ./mill run
+[10 20 30 40 50]
+
+*/

example/pythonlib/dependencies/2-pip-requirements/requirements.txt

@@ -0,0 +1 @@
+numpy==2.1.2

example/pythonlib/dependencies/2-pip-requirements/src/main.py

@@ -0,0 +1,4 @@
+import numpy as np
+
+data = np.array([10, 20, 30, 40, 50])
+print(data)

example/pythonlib/dependencies/3-unmanaged-wheels/build.mill

@@ -0,0 +1,25 @@
+// In most scenarios you should rely on `pythonDeps`/`moduleDeps` and let Mill
+// manage the downloading and caching of wheels for you. But in the rare case
+// you receive a wheel or folder-full-of-wheels from somewhere and need to
+// include it in your project, `unmanagedWheels` is the way to do it.
+
+package build
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule {
+  def unmanagedWheels: T[Seq[PathRef]] = Task.Input {
+    Seq.from(os.list(millSourcePath / "lib").map(PathRef(_)))
+  }
+}
+
+// You can override `unmanagedWheels` to point it at a wheel (.whl file) or
+// source distribution (.tar.gz with a pyproject.toml file) you place on the
+// filesystem, e.g. in the above snippet any files that happen to live in the
+// `lib/` folder.
+
+/** Usage
+
+> ./mill run
+b'"Hello, world!"'
+
+*/

example/pythonlib/dependencies/3-unmanaged-wheels/src/main.py

@@ -0,0 +1,9 @@
+# this comes from a wheel
+import jinja2
+
+# this comes from an sdist
+import orjson as oj
+
+environment = jinja2.Environment()
+template = environment.from_string("Hello, {{ name }}!")
+print(oj.dumps(template.render(name="world")))

example/pythonlib/dependencies/4-downloading-unmanaged-wheels/build.mill

@@ -0,0 +1,37 @@
+// You can also override `unmanagedWheels` to point it at wheels that you want to
+// download from arbitrary URLs.
+// `requests.get` comes from the https://github.com/com-lihaoyi/requests-scala[Requests-Scala]
+// library, one of Mill's xref:fundamentals/bundled-libraries.adoc[Bundled Libraries].
+//
+package build
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule {
+  def unmanagedWheels = Task {
+    val name = "jinja2-3.1.4-py3-none-any.whl"
+    val url = s"https://github.com/pallets/jinja/releases/download/3.1.4/$name"
+    os.write(Task.dest / name, requests.get.stream(url))
+    Seq(PathRef(Task.dest / name))
+  }
+}
+
+/** Usage
+
+> ./mill run
+Hello, world!
+
+*/
+
+// Tasks like `unmanagedWheels` and `pythonDeps` are cached, so your wheel is downloaded only
+// once and re-used indefinitely after that. This is usually not a problem, because usually URLs
+// follow the rule that https://www.w3.org/Provider/Style/URI[Cool URIs don't change], and so files
+// downloaded from the same URL will always contain the same contents.
+//
+// NOTE: An unmanaged wheel downloaded via `requests.get` is still unmanaged: even though you
+// downloaded it from somewhere, `requests.get` does not know how to pull in third party
+// dependencies or de-duplicate different versions on the classpath. All the same caveats you need
+// to worry about when dealing with xref:#_unmanaged_wheels[unmanaged wheels] apply here as well. In
+// case you **do** want mill to take care of managing dependencies of a package which is not
+// available on PyPI, you shouldn't get that package in `unmanagedWheels` (like we did in the
+// example above). Instead, you can declare the dependency as a regular `pythonDep`
+// https://peps.python.org/pep-0440/#direct-references[as a direct URL that pip understands].

example/pythonlib/dependencies/4-downloading-unmanaged-wheels/src/main.py

@@ -0,0 +1,4 @@
+import jinja2
+environment = jinja2.Environment()
+template = environment.from_string("Hello, {{ name }}!")
+print(template.render(name="world"))

example/pythonlib/dependencies/5-repository-config/build.mill

@@ -0,0 +1,44 @@
+// By default, dependencies are resolved from https://pypi.org/[the Python
+// Package Index (PyPI)], the standard package index for python projects. You
+// can also add your own package indexes by overriding the `indexes` task in
+// the module:
+
+package build
+import mill._, pythonlib._
+
+object foo extends PythonModule {
+
+  def pythonDeps = Seq(
+    "testpkg-jodersky==0.0.1" // a test package, only available on test.pypi.org
+  )
+
+  // override this task to add or replace the package indexes
+  def indexes = super.indexes() ++ Seq("https://test.pypi.org/simple/")
+}
+
+// Mill uses https://pip.pypa.io/en/stable/[pip] to find and install dependencies.
+//
+// You can configure pip through its
+// https://pip.pypa.io/en/stable/topics/configuration/#location[normal configuration files.]
+//
+// === Private indexes
+//
+// You can read up in more detail on https://pip.pypa.io/en/stable/topics/authentication/[how to
+// configure pip to authenticate to private indexes]. Here is an example which reads a package from
+// an environment variable:
+
+object bar extends PythonModule {
+  def indexPassword = Task.Input { Task.env.apply("COMPANY_PASSWORD") }
+  def indexes = Task {
+    Seq(s"https://username:${indexPassword()}@pypi.company.com/simple")
+  }
+}
+
+// More advanced authentication techniques are available by configuring pip directly.
+
+/** Usage
+
+> ./mill foo.run
+2
+
+*/

example/pythonlib/dependencies/5-repository-config/foo/src/main.py

@@ -0,0 +1,3 @@
+from testpkg_jodersky import example
+
+print(example.add_one(1))

example/pythonlib/dependencies/6-debugging/build.mill

@@ -0,0 +1,34 @@
+// In case anything goes wrong, or if you're just curious, you can see what
+// arguments mill passes to `pip install` by looking at the output of the
+// `pipInstallArgs` task.
+
+package build
+import mill._, pythonlib._
+
+object `package` extends RootModule with PythonModule {
+  def pythonDeps = Seq(
+    "numpy==2.1.2",
+    "pandas~=2.2.3",
+    "jinja2 @ https://github.com/pallets/jinja/releases/download/3.1.4/jinja2-3.1.4-py3-none-any.whl"
+  )
+
+  def indexes = Seq("invalid_index")
+}
+
+/** Usage
+
+> ./mill show pipInstallArgs
+{
+  "args": [
+    "--index-url",
+    "invalid_index",
+    "mypy==1.13.0",
+    "pex==2.24.1",
+    "numpy==2.1.2",
+    "pandas~=2.2.3",
+    "jinja2 @ https://github.com/pallets/jinja/releases/download/3.1.4/jinja2-3.1.4-py3-none-any.whl"
+  ],
+  "sig": ...
+}
+
+*/