docs(): generate docs for our Python SDK (#7612)

docs-website/README.md

@@ -136,3 +136,15 @@ The purpose of this section is to provide developers & technical users with conc
 **Feature Guides**
 
 This section aims to provide plain-language feature overviews for both technical and non-technical readers alike.
+
+
+## Docs site generation process
+
+This process is orchestrated by a combination of Gradle and Yarn tasks. The main entrypoint is via the `docs-website:yarnGenerate` task, which in turn eventually runs `yarn run generate`.
+
+Steps:
+1. Generate the GraphQL combined schema using the gradle's `docs-website:generateGraphQLSchema` task. This generates `./graphql/combined.graphql`.
+2. Run `yarn run _generate-graphql` to produce some markdown in the `./docs` directory.
+3. Run the `generateDocsDir.ts` script to add the remaining markdown files to the `./docs` directory.
+4. Run a copy or rsync to copy the `./docs` directory to `./genDocs`.
+5. The docusaurus build process will then use the `./genDocs` directory as the source for the docs site.

docs-website/package.json

@@ -10,8 +10,9 @@
     "deploy": "docusaurus deploy",
     "serve": "docusaurus serve",
     "clear": "docusaurus clear && rm -rf genDocs/*",
-    "_generate-graphql": "mkdir -p genDocs && docusaurus docs:generate:graphql",
-    "_generate-docs": "rm -rf docs && mkdir docs && yarn _generate-graphql && ts-node -O '{ \"lib\": [\"es2020\"], \"target\": \"es6\" }' generateDocsDir.ts",
+    "_generate-graphql": "docusaurus docs:generate:graphql",
+    "_generate-python-sdk": "cd sphinx && make md",
+    "_generate-docs": "rm -rf docs && mkdir docs && yarn _generate-graphql && yarn _generate-python-sdk && ts-node -O '{ \"lib\": [\"es2020\"], \"target\": \"es6\" }' generateDocsDir.ts",
     "generate": "rm -rf genDocs genStatic && mkdir genDocs genStatic && yarn _generate-docs && mv docs/* genDocs/ && rmdir docs",
     "generate-rsync": "mkdir -p genDocs genStatic && yarn _generate-docs && rsync -v --checksum -r -h -i --delete docs/ genDocs && rm -rf docs",
     "lint": "prettier -w generateDocsDir.ts sidebars.js src/pages/index.js",

docs-website/sidebars.js

@@ -416,6 +416,14 @@ module.exports = {
             "metadata-integration/java/as-a-library",
           ],
         },
+        {
+          "Python SDK Reference": [
+            {
+              type: "autogenerated",
+              dirName: "python-sdk",
+            },
+          ],
+        },
         "docs/datahub_lite",
       ],
     },

docs-website/sphinx/.gitignore

@@ -0,0 +1,4 @@
+
+/.envrc
+/venv
+/_build

docs-website/sphinx/Makefile

@@ -0,0 +1,37 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+VENV_DIR	  ?= venv
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= $(VENV_DIR)/bin/sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+VENV_SENTINEL = $(VENV_DIR)/.sentinel
+
+# Put it first so that "make" without argument is like "make help".
+help: venv
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+venv: $(VENV_SENTINEL)
+
+$(VENV_SENTINEL): requirements.txt
+	python3 -m venv $(VENV_DIR)
+	$(VENV_DIR)/bin/pip install --upgrade pip wheel setuptools
+	$(VENV_DIR)/bin/pip install -r requirements.txt
+	touch $(VENV_SENTINEL)
+
+.PHONY: help html doctest linkcheck clean serve md
+
+# Not using Python's http.server because it enables caching headers.
+serve:
+	serve -p 3001 _build/html/
+
+md: html
+	$(VENV_DIR)/bin/python3 convert_sphinx_to_docusaurus.py
+
+# Route other targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+html doctest linkcheck clean: venv Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs-website/sphinx/apidocs/builder.rst

@@ -0,0 +1,10 @@
+Builder
+=======
+
+These classes and methods make it easier to construct MetadataChangeProposals and MetadataChangeEvents.
+
+.. automodule:: datahub.emitter.mcp
+
+.. automodule:: datahub.emitter.mce_builder
+
+.. automodule:: datahub.emitter.mcp_builder

docs-website/sphinx/apidocs/clients.rst

@@ -0,0 +1,11 @@
+Client
+======
+
+The Kafka emitter or Rest emitter can be used to push metadata to DataHub.
+The DataHub graph client extends the Rest emitter with additional functionality.
+
+.. automodule:: datahub.emitter.rest_emitter
+
+.. automodule:: datahub.emitter.kafka_emitter
+
+.. automodule:: datahub.ingestion.graph.client

docs-website/sphinx/apidocs/models.rst

@@ -0,0 +1,6 @@
+Models
+======
+
+.. automodule:: datahub.metadata.schema_classes
+   :exclude-members: construct_with_defaults, RECORD_SCHEMA, ASPECT_NAME, ASPECT_INFO
+   :member-order: alphabetical

docs-website/sphinx/basic.css

@@ -0,0 +1,261 @@
+/*
+ * basic.css
+ * ~~~~~~~~~
+ *
+ * Sphinx stylesheet -- basic theme.
+ *
+ * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ * DataHub Modifications:
+ * - Removed most styles, which weren't super relevant.
+ * - Remove font-weight on dl.field-list > dt
+ * - Remove highlight-related styles.
+ *
+ */
+
+/* -- field list styles ----------------------------------------------------- */
+
+table.field-list td, table.field-list th {
+    border: 0 !important;
+}
+
+.field-list ul {
+    margin: 0;
+    padding-left: 1em;
+}
+
+.field-list p {
+    margin: 0;
+}
+
+.field-name {
+    -moz-hyphens: manual;
+    -ms-hyphens: manual;
+    -webkit-hyphens: manual;
+    hyphens: manual;
+}
+
+/* -- hlist styles ---------------------------------------------------------- */
+
+table.hlist {
+    margin: 1em 0;
+}
+
+table.hlist td {
+    vertical-align: top;
+}
+
+/* -- object description styles --------------------------------------------- */
+
+.sig {
+	font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+}
+
+.sig-name, code.descname {
+    background-color: transparent;
+    font-weight: bold;
+}
+
+.sig-name {
+	font-size: 1.1em;
+}
+
+code.descname {
+    font-size: 1.2em;
+}
+
+.sig-prename, code.descclassname {
+    background-color: transparent;
+}
+
+.optional {
+    font-size: 1.3em;
+}
+
+.sig-paren {
+    font-size: larger;
+}
+
+.sig-param.n {
+	font-style: italic;
+}
+
+/* C++ specific styling */
+
+.sig-inline.c-texpr,
+.sig-inline.cpp-texpr {
+	font-family: unset;
+}
+
+.sig.c   .k, .sig.c   .kt,
+.sig.cpp .k, .sig.cpp .kt {
+	color: #0033B3;
+}
+
+.sig.c   .m,
+.sig.cpp .m {
+	color: #1750EB;
+}
+
+.sig.c   .s, .sig.c   .sc,
+.sig.cpp .s, .sig.cpp .sc {
+	color: #067D17;
+}
+
+
+/* -- other body styles ----------------------------------------------------- */
+
+ol.arabic {
+    list-style: decimal;
+}
+
+ol.loweralpha {
+    list-style: lower-alpha;
+}
+
+ol.upperalpha {
+    list-style: upper-alpha;
+}
+
+ol.lowerroman {
+    list-style: lower-roman;
+}
+
+ol.upperroman {
+    list-style: upper-roman;
+}
+
+:not(li) > ol > li:first-child > :first-child,
+:not(li) > ul > li:first-child > :first-child {
+    margin-top: 0px;
+}
+
+:not(li) > ol > li:last-child > :last-child,
+:not(li) > ul > li:last-child > :last-child {
+    margin-bottom: 0px;
+}
+
+ol.simple ol p,
+ol.simple ul p,
+ul.simple ol p,
+ul.simple ul p {
+    margin-top: 0;
+}
+
+ol.simple > li:not(:first-child) > p,
+ul.simple > li:not(:first-child) > p {
+    margin-top: 0;
+}
+
+ol.simple p,
+ul.simple p {
+    margin-bottom: 0;
+}
+
+aside.footnote > span,
+div.citation > span {
+    float: left;
+}
+aside.footnote > span:last-of-type,
+div.citation > span:last-of-type {
+  padding-right: 0.5em;
+}
+aside.footnote > p {
+  margin-left: 2em;
+}
+div.citation > p {
+  margin-left: 4em;
+}
+aside.footnote > p:last-of-type,
+div.citation > p:last-of-type {
+    margin-bottom: 0em;
+}
+aside.footnote > p:last-of-type:after,
+div.citation > p:last-of-type:after {
+    content: "";
+    clear: both;
+}
+
+dl.field-list {
+    display: grid;
+    grid-template-columns: fit-content(30%) auto;
+}
+
+dl.field-list > dt {
+    /* font-weight: bold; */
+    word-break: break-word;
+    padding-left: 0.5em;
+    padding-right: 5px;
+}
+
+dl.field-list > dd {
+    padding-left: 0.5em;
+    margin-top: 0em;
+    margin-left: 0em;
+    margin-bottom: 0em;
+}
+
+dl {
+    margin-bottom: 15px;
+}
+
+dd > :first-child {
+    margin-top: 0px;
+}
+
+dd ul, dd table {
+    margin-bottom: 10px;
+}
+
+dd {
+    margin-top: 3px;
+    margin-bottom: 10px;
+    margin-left: 30px;
+}
+
+dl > dd:last-child,
+dl > dd:last-child > :last-child {
+    margin-bottom: 0;
+}
+
+dl.glossary dt {
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+.line-block {
+    display: block;
+    margin-top: 1em;
+    margin-bottom: 1em;
+}
+
+.line-block .line-block {
+    margin-top: 0;
+    margin-bottom: 0;
+    margin-left: 1.5em;
+}
+
+.guilabel, .menuselection {
+    font-family: sans-serif;
+}
+
+.accelerator {
+    text-decoration: underline;
+}
+
+.classifier {
+    font-style: oblique;
+}
+
+.classifier:before {
+    font-style: normal;
+    margin: 0 0.5em;
+    content: ":";
+    display: inline-block;
+}
+
+abbr, acronym {
+    border-bottom: dotted 1px;
+    cursor: help;
+}

docs-website/sphinx/cli.rst

@@ -0,0 +1,6 @@
+DataHub CLI
+===========
+
+.. click:: datahub.entrypoints:datahub
+   :prog: datahub
+   :nested: full