diff --git a/docs/website/.gitignore b/docs/website/.gitignore index bc94ffc693..861b383383 100644 --- a/docs/website/.gitignore +++ b/docs/website/.gitignore @@ -23,3 +23,9 @@ jaffle_shop npm-debug.log* yarn-debug.log* yarn-error.log* + +# ignore all versions, there are generated dynamically +versions.json +versioned_docs +versioned_sidebars +.dlt-repo \ No newline at end of file diff --git a/docs/website/README.md b/docs/website/README.md index b907d1380a..31bdbdde07 100644 --- a/docs/website/README.md +++ b/docs/website/README.md @@ -64,4 +64,29 @@ The site is deployed using `netlify`. The `netlify` build command is: npm run build:netlify ``` -It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`. \ No newline at end of file +It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`. + +## Docs versions + +We keep a few additional versions of our docs for the users to be able read about how former and future versions of dlt work. We use docusaurus versions for this but we do not check the historical versions into the repo but rather use a script to build the former versions on deployment. To locally build the versions run: + +``` +npm run update-versions +``` + +This will execute the script at tools/update_versions.js. This tool will do the following: + +* Find all the highest minor versions the tags of the repo (e.g. 0.4.13, 0.5.22, 1.1.3) +* It will create a version for all of these tags that are larger than the minimum version defined in MINIMUM_SEMVER_VERSION in the script. +* It will NOT create a version for the highest version, we assume the most up to date docs for the highest versions are the tip of master +* It will NOT create any docs versions for pre-releases. +* It will create a future version called "devel" from the current commit of this repo. +* It will set up docusaurus to display all of these versions correctly. + +You can clear these versions with + +``` +npm run clear-versions +``` + +The netflify deployment of these docs need to happen from the master branch so that the current version gets properly selected. \ No newline at end of file diff --git a/docs/website/docusaurus.config.js b/docs/website/docusaurus.config.js index d011de95b3..72459606f6 100644 --- a/docs/website/docusaurus.config.js +++ b/docs/website/docusaurus.config.js @@ -1,11 +1,43 @@ // @ts-check // Note: type annotations allow type checking and IDEs autocompletion +const fs = require("fs") require('dotenv').config() const lightCodeTheme = require('prism-react-renderer/themes/dracula'); // const lightCodeTheme = require('prism-react-renderer/themes/github'); const darkCodeTheme = require('prism-react-renderer/themes/dracula'); +// create versions config +const versions = {"current": { + label: 'devel', + path: 'devel', + noIndex: true +}} + +// inject master version renaming only if versions present +if (fs.existsSync("versions.json")) { + let latestLabel = "latest" + if (process.env.DOCUSAURUS_DLT_VERSION) { + latestLabel = `${process.env.DOCUSAURUS_DLT_VERSION} (latest)` + } + + + versions["master"] = { + label: latestLabel, + path: '/' + } + // disable indexing for all known versions + for (let v of JSON.parse(fs.readFileSync("versions.json"))) { + if (v == "master") { + continue; + } + versions[v] = { + noIndex: true + } + } + +} + /** @type {import('@docusaurus/types').Config} */ const config = { title: 'dlt Docs', @@ -30,8 +62,6 @@ const config = { locales: ['en'], }, - - presets: [ [ '@docusaurus/preset-classic', @@ -50,12 +80,7 @@ const config = { editUrl: (params) => { return "https://github.com/dlt-hub/dlt/tree/devel/docs/website/docs/" + params.docPath; }, - versions: { - current: { - label: 'current', - }, - }, - lastVersion: 'current', + versions: versions, showLastUpdateAuthor: true, showLastUpdateTime: true, }, @@ -83,7 +108,9 @@ const config = { href: 'https://dlthub.com' }, items: [ - { label: 'dlt ' + (process.env.IS_MASTER_BRANCH ? "stable ": "devel ") + (process.env.DOCUSAURUS_DLT_VERSION || "0.0.1"), position: 'left', href: 'https://github.com/dlt-hub/dlt', className: 'version-navbar' }, + { + type: 'docsVersionDropdown', + }, { type: 'doc', docId: 'intro', @@ -196,15 +223,4 @@ const config = { ], }; -if (!process.env.IS_MASTER_BRANCH && config.themeConfig) { - config.themeConfig.announcementBar = { - id: 'devel docs', - content: - 'This is the development version of the dlt docs. Go to the stable docs.', - backgroundColor: '#4c4898', - textColor: '#fff', - isCloseable: false, - } -} - module.exports = config; diff --git a/docs/website/package-lock.json b/docs/website/package-lock.json index f99a70dbf7..f1409905f1 100644 --- a/docs/website/package-lock.json +++ b/docs/website/package-lock.json @@ -20,6 +20,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-twitter-embed": "^4.0.4", + "semver": "^7.6.3", "string-dedent": "^3.0.1", "sync-fetch": "^0.5.2", "toml": "^3.0.0" @@ -11075,12 +11076,10 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -11107,22 +11106,6 @@ "semver": "bin/semver.js" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/send": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", diff --git a/docs/website/package.json b/docs/website/package.json index becf1c8bc6..d904c8d77b 100644 --- a/docs/website/package.json +++ b/docs/website/package.json @@ -4,16 +4,18 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"", - "build": "node tools/preprocess_docs.js && PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && docusaurus build", - "build:netlify": "node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && node tools/update_version_env.js && docusaurus build --out-dir build/docs", + "start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"", + "build": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. poetry run pydoc-markdown && docusaurus build", + "build:netlify": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && docusaurus build --out-dir build/docs", "swizzle": "docusaurus swizzle", "clear": "docusaurus clear", "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", "preprocess-docs": "node tools/preprocess_docs.js", - "generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown" + "generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown", + "clear-versions": "node tools/clear_versions.js", + "update-versions": "node tools/update_versions.js" }, "dependencies": { "@docusaurus/core": "2.4.3", @@ -28,6 +30,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-twitter-embed": "^4.0.4", + "semver": "^7.6.3", "string-dedent": "^3.0.1", "sync-fetch": "^0.5.2", "toml": "^3.0.0" diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index 921c3c0dc4..56c2eb165c 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -356,21 +356,4 @@ if (fs.existsSync('./docs_processed/api_reference/sidebar.json')) { } } -// on the master branch link to devel and vice versa -if (process.env.IS_MASTER_BRANCH) { - sidebars.tutorialSidebar.push( { - type: 'link', - label: 'Switch to Devel Docs', - href: 'https://dlthub.com/devel/intro', - className: 'learn-more-link', - }) -} else { - sidebars.tutorialSidebar.push( { - type: 'link', - label: 'Switch to Stable Docs', - href: 'https://dlthub.com/docs/intro', - className: 'learn-more-link', - }) -} - module.exports = sidebars; diff --git a/docs/website/tools/clear_versions.js b/docs/website/tools/clear_versions.js new file mode 100644 index 0000000000..e66f5ed823 --- /dev/null +++ b/docs/website/tools/clear_versions.js @@ -0,0 +1,11 @@ +const fs = require('fs'); + +version_files = [ + "versions.json", + "versioned_docs", + "versioned_sidebars" +] + +for (let f of version_files) { + fs.rmSync(f, { recursive: true, force: true }) +} \ No newline at end of file diff --git a/docs/website/tools/update_version_env.js b/docs/website/tools/update_version_env.js deleted file mode 100644 index 15d10bd970..0000000000 --- a/docs/website/tools/update_version_env.js +++ /dev/null @@ -1,17 +0,0 @@ -// creates an .env file with the current version of the dlt for consumption in docusaurus.config.js - -const { readFile, writeFile } = require('fs/promises') -const tom = require('toml'); - -const TOML_FILE = '../../pyproject.toml'; -const ENV_FILE = '.env' - -async function update_env() { - const fileContent = await readFile(TOML_FILE, 'utf8'); - const toml = tom.parse(fileContent); - const version = toml['tool']['poetry']['version']; - const envFileContent = `DOCUSAURUS_DLT_VERSION=${version}`; - await writeFile(ENV_FILE, envFileContent, 'utf8'); -} - -update_env(); \ No newline at end of file diff --git a/docs/website/tools/update_versions.js b/docs/website/tools/update_versions.js new file mode 100644 index 0000000000..855766c5dd --- /dev/null +++ b/docs/website/tools/update_versions.js @@ -0,0 +1,107 @@ +const proc = require('child_process') +const fs = require('fs'); +const semver = require('semver') + + +// const +const REPO_DIR = ".dlt-repo" +const REPO_DOCS_DIR = REPO_DIR + "/docs/website" +const REPO_URL = "https://github.com/dlt-hub/dlt.git" +const VERSIONED_DOCS_FOLDER = "versioned_docs" +const VERSIONED_SIDEBARS_FOLDER = "versioned_sidebars" +const ENV_FILE = '.env' + +// no doc versions below this version will be deployed +const MINIMUM_SEMVER_VERSION = "0.5.0" + +// clear old repo version +fs.rmSync(REPO_DIR, { recursive: true, force: true }) + +// checkout fresh +console.log("Checking out dlt repo") +proc.execSync(`git clone ${REPO_URL} ${REPO_DIR}`) + +// find tags +console.log("Discovering versions") +const tags = proc.execSync(`cd ${REPO_DIR} && git tag`).toString().trim().split("\n"); +console.log(`Found ${tags.length} tags`) + +// parse and filter invalid tags +let versions = tags.map(v => semver.valid(v)).filter(v => v != null) + +// remove all tags below the min version and sort +min_version = semver.valid(MINIMUM_SEMVER_VERSION) +versions = semver.rsort(versions.filter(v => semver.gt(v, min_version))) + +// remove prelease versions +versions.filter(v => semver.prerelease(v) == null) + +console.log(`Found ${versions.length} elligible versions`) +if (versions.length < 2) { + console.error("Sanity check failed, not enough elligble version tags found") + process.exit(1) +} + +// write last version into env file +const envFileContent = `DOCUSAURUS_DLT_VERSION=${versions[0]}`; +fs.writeFileSync(ENV_FILE, envFileContent, 'utf8'); + +// go through the versions and find all newest versions of any major version +// the newest version is replace by the master branch here so the master head +// always is the "current" doc +const selectedVersions = ["master"]; +let lastVersion = versions[0]; +for (let ver of versions) { + if ( semver.major(ver) != semver.major(lastVersion)) { + selectedVersions.push(ver) + } + lastVersion = ver; +} + +console.log(`Will create docs versions for ${selectedVersions}`) + +// create folders +fs.rmSync(VERSIONED_DOCS_FOLDER, { recursive: true, force: true }) +fs.rmSync(VERSIONED_SIDEBARS_FOLDER, { recursive: true, force: true }) +fs.rmSync("versions.json", { force: true }) + +fs.mkdirSync(VERSIONED_DOCS_FOLDER); +fs.mkdirSync(VERSIONED_SIDEBARS_FOLDER); + +// check that checked out repo is on devel +console.log("Checking branch") +const branch = proc.execSync(`cd ${REPO_DIR} && git rev-parse --abbrev-ref HEAD`).toString().trim() + +// sanity check +if (branch != "devel") { + console.error("Could not check out devel branch") + process.exit(1) +} + +selectedVersions.reverse() +for (const version of selectedVersions) { + + // checkout verison and verify we have the right tag + console.log(`Generating version ${version}, switching to tag:`) + proc.execSync(`cd ${REPO_DIR} && git checkout ${version}`) + + // const tag = proc.execSync(`cd ${REPO_DIR} && git describe --exact-match --tags`).toString().trim() + // if (tag != version) { + // console.error(`Could not checkout version ${version}`) + // process.exit(1) + // } + + // build doc version, we also run preprocessing and markdown gen for each doc version + console.log(`Building docs...`) + proc.execSync(`cd ${REPO_DOCS_DIR} && npm run preprocess-docs && PYTHONPATH=. pydoc-markdown`) + + console.log(`Snapshotting version...`) + proc.execSync(`cd ${REPO_DOCS_DIR} && npm run docusaurus docs:version ${version}`) + + console.log(`Moving snapshot`) + fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_DOCS_FOLDER, VERSIONED_DOCS_FOLDER, {recursive: true}) + fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_SIDEBARS_FOLDER, VERSIONED_SIDEBARS_FOLDER, {recursive: true}) + +} + +fs.cpSync(REPO_DOCS_DIR+"/versions.json", "versions.json")