Skip to content

Commit

Permalink
Add historic docs versions
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Sep 6, 2024
1 parent e9c9ecf commit cb9c44c
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 81 deletions.
6 changes: 6 additions & 0 deletions docs/website/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,9 @@ jaffle_shop
npm-debug.log*
yarn-debug.log*
yarn-error.log*

# ignore all versions, there are generated dynamically
versions.json
versioned_docs
versioned_sidebars
.dlt-repo
27 changes: 26 additions & 1 deletion docs/website/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,29 @@ The site is deployed using `netlify`. The `netlify` build command is:
npm run build:netlify
```

It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`.
It will place the build in `build/docs` folder. The `netlify.toml` redirects from root path `/` into `/docs`.

## Docs versions

We keep a few additional versions of our docs for the users to be able read about how former and future versions of dlt work. We use docusaurus versions for this but we do not check the historical versions into the repo but rather use a script to build the former versions on deployment. To locally build the versions run:

```
npm run update-versions
```

This will execute the script at tools/update_versions.js. This tool will do the following:

* Find all the highest minor versions the tags of the repo (e.g. 0.4.13, 0.5.22, 1.1.3)
* It will create a version for all of these tags that are larger than the minimum version defined in MINIMUM_SEMVER_VERSION in the script.
* It will NOT create a version for the highest version, we assume the most up to date docs for the highest versions are the tip of master
* It will NOT create any docs versions for pre-releases.
* It will create a future version called "devel" from the current commit of this repo.
* It will set up docusaurus to display all of these versions correctly.

You can clear these versions with

```
npm run clear-versions
```

The netflify deployment of these docs need to happen from the master branch so that the current version gets properly selected.
56 changes: 36 additions & 20 deletions docs/website/docusaurus.config.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,43 @@
// @ts-check
// Note: type annotations allow type checking and IDEs autocompletion
const fs = require("fs")
require('dotenv').config()

const lightCodeTheme = require('prism-react-renderer/themes/dracula');
// const lightCodeTheme = require('prism-react-renderer/themes/github');
const darkCodeTheme = require('prism-react-renderer/themes/dracula');

// create versions config
const versions = {"current": {
label: 'devel',
path: 'devel',
noIndex: true
}}

// inject master version renaming only if versions present
if (fs.existsSync("versions.json")) {
let latestLabel = "latest"
if (process.env.DOCUSAURUS_DLT_VERSION) {
latestLabel = `${process.env.DOCUSAURUS_DLT_VERSION} (latest)`
}


versions["master"] = {
label: latestLabel,
path: '/'
}
// disable indexing for all known versions
for (let v of JSON.parse(fs.readFileSync("versions.json"))) {
if (v == "master") {
continue;
}
versions[v] = {
noIndex: true
}
}

}

/** @type {import('@docusaurus/types').Config} */
const config = {
title: 'dlt Docs',
Expand All @@ -30,8 +62,6 @@ const config = {
locales: ['en'],
},



presets: [
[
'@docusaurus/preset-classic',
Expand All @@ -50,12 +80,7 @@ const config = {
editUrl: (params) => {
return "https://github.com/dlt-hub/dlt/tree/devel/docs/website/docs/" + params.docPath;
},
versions: {
current: {
label: 'current',
},
},
lastVersion: 'current',
versions: versions,
showLastUpdateAuthor: true,
showLastUpdateTime: true,
},
Expand Down Expand Up @@ -83,7 +108,9 @@ const config = {
href: 'https://dlthub.com'
},
items: [
{ label: 'dlt ' + (process.env.IS_MASTER_BRANCH ? "stable ": "devel ") + (process.env.DOCUSAURUS_DLT_VERSION || "0.0.1"), position: 'left', href: 'https://github.com/dlt-hub/dlt', className: 'version-navbar' },
{
type: 'docsVersionDropdown',
},
{
type: 'doc',
docId: 'intro',
Expand Down Expand Up @@ -196,15 +223,4 @@ const config = {
],
};

if (!process.env.IS_MASTER_BRANCH && config.themeConfig) {
config.themeConfig.announcementBar = {
id: 'devel docs',
content:
'This is the development version of the dlt docs. <a target="_blank" rel="noopener noreferrer" href="https://dlthub.com/docs/intro">Go to the stable docs.</a>',
backgroundColor: '#4c4898',
textColor: '#fff',
isCloseable: false,
}
}

module.exports = config;
27 changes: 5 additions & 22 deletions docs/website/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions docs/website/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,18 @@
"private": true,
"scripts": {
"docusaurus": "docusaurus",
"start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"",
"build": "node tools/preprocess_docs.js && PYTHONPATH=. poetry run pydoc-markdown && node tools/update_version_env.js && docusaurus build",
"build:netlify": "node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && node tools/update_version_env.js && docusaurus build --out-dir build/docs",
"start": "PYTHONPATH=. poetry run pydoc-markdown && node tools/preprocess_docs.js && concurrently --kill-others \"node tools/preprocess_docs.js --watch\" \"docusaurus start\"",
"build": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. poetry run pydoc-markdown && docusaurus build",
"build:netlify": "node tools/update_versions.js && node tools/preprocess_docs.js && PYTHONPATH=. pydoc-markdown && docusaurus build --out-dir build/docs",
"swizzle": "docusaurus swizzle",
"clear": "docusaurus clear",
"serve": "docusaurus serve",
"write-translations": "docusaurus write-translations",
"write-heading-ids": "docusaurus write-heading-ids",
"preprocess-docs": "node tools/preprocess_docs.js",
"generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown"
"generate-api-reference": "PYTHONPATH=. poetry run pydoc-markdown",
"clear-versions": "node tools/clear_versions.js",
"update-versions": "node tools/update_versions.js"
},
"dependencies": {
"@docusaurus/core": "2.4.3",
Expand All @@ -28,6 +30,7 @@
"react": "^17.0.2",
"react-dom": "^17.0.2",
"react-twitter-embed": "^4.0.4",
"semver": "^7.6.3",
"string-dedent": "^3.0.1",
"sync-fetch": "^0.5.2",
"toml": "^3.0.0"
Expand Down
17 changes: 0 additions & 17 deletions docs/website/sidebars.js
Original file line number Diff line number Diff line change
Expand Up @@ -356,21 +356,4 @@ if (fs.existsSync('./docs_processed/api_reference/sidebar.json')) {
}
}

// on the master branch link to devel and vice versa
if (process.env.IS_MASTER_BRANCH) {
sidebars.tutorialSidebar.push( {
type: 'link',
label: 'Switch to Devel Docs',
href: 'https://dlthub.com/devel/intro',
className: 'learn-more-link',
})
} else {
sidebars.tutorialSidebar.push( {
type: 'link',
label: 'Switch to Stable Docs',
href: 'https://dlthub.com/docs/intro',
className: 'learn-more-link',
})
}

module.exports = sidebars;
11 changes: 11 additions & 0 deletions docs/website/tools/clear_versions.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const fs = require('fs');

version_files = [
"versions.json",
"versioned_docs",
"versioned_sidebars"
]

for (let f of version_files) {
fs.rmSync(f, { recursive: true, force: true })
}
17 changes: 0 additions & 17 deletions docs/website/tools/update_version_env.js

This file was deleted.

107 changes: 107 additions & 0 deletions docs/website/tools/update_versions.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
const proc = require('child_process')
const fs = require('fs');
const semver = require('semver')


// const
const REPO_DIR = ".dlt-repo"
const REPO_DOCS_DIR = REPO_DIR + "/docs/website"
const REPO_URL = "https://github.com/dlt-hub/dlt.git"
const VERSIONED_DOCS_FOLDER = "versioned_docs"
const VERSIONED_SIDEBARS_FOLDER = "versioned_sidebars"
const ENV_FILE = '.env'

// no doc versions below this version will be deployed
const MINIMUM_SEMVER_VERSION = "0.5.0"

// clear old repo version
fs.rmSync(REPO_DIR, { recursive: true, force: true })

// checkout fresh
console.log("Checking out dlt repo")
proc.execSync(`git clone ${REPO_URL} ${REPO_DIR}`)

// find tags
console.log("Discovering versions")
const tags = proc.execSync(`cd ${REPO_DIR} && git tag`).toString().trim().split("\n");
console.log(`Found ${tags.length} tags`)

// parse and filter invalid tags
let versions = tags.map(v => semver.valid(v)).filter(v => v != null)

// remove all tags below the min version and sort
min_version = semver.valid(MINIMUM_SEMVER_VERSION)
versions = semver.rsort(versions.filter(v => semver.gt(v, min_version)))

// remove prelease versions
versions.filter(v => semver.prerelease(v) == null)

console.log(`Found ${versions.length} elligible versions`)
if (versions.length < 2) {
console.error("Sanity check failed, not enough elligble version tags found")
process.exit(1)
}

// write last version into env file
const envFileContent = `DOCUSAURUS_DLT_VERSION=${versions[0]}`;
fs.writeFileSync(ENV_FILE, envFileContent, 'utf8');

// go through the versions and find all newest versions of any major version
// the newest version is replace by the master branch here so the master head
// always is the "current" doc
const selectedVersions = ["master"];
let lastVersion = versions[0];
for (let ver of versions) {
if ( semver.major(ver) != semver.major(lastVersion)) {
selectedVersions.push(ver)
}
lastVersion = ver;
}

console.log(`Will create docs versions for ${selectedVersions}`)

// create folders
fs.rmSync(VERSIONED_DOCS_FOLDER, { recursive: true, force: true })
fs.rmSync(VERSIONED_SIDEBARS_FOLDER, { recursive: true, force: true })
fs.rmSync("versions.json", { force: true })

fs.mkdirSync(VERSIONED_DOCS_FOLDER);
fs.mkdirSync(VERSIONED_SIDEBARS_FOLDER);

// check that checked out repo is on devel
console.log("Checking branch")
const branch = proc.execSync(`cd ${REPO_DIR} && git rev-parse --abbrev-ref HEAD`).toString().trim()

// sanity check
if (branch != "devel") {
console.error("Could not check out devel branch")
process.exit(1)
}

selectedVersions.reverse()
for (const version of selectedVersions) {

// checkout verison and verify we have the right tag
console.log(`Generating version ${version}, switching to tag:`)
proc.execSync(`cd ${REPO_DIR} && git checkout ${version}`)

// const tag = proc.execSync(`cd ${REPO_DIR} && git describe --exact-match --tags`).toString().trim()
// if (tag != version) {
// console.error(`Could not checkout version ${version}`)
// process.exit(1)
// }

// build doc version, we also run preprocessing and markdown gen for each doc version
console.log(`Building docs...`)
proc.execSync(`cd ${REPO_DOCS_DIR} && npm run preprocess-docs && PYTHONPATH=. pydoc-markdown`)

console.log(`Snapshotting version...`)
proc.execSync(`cd ${REPO_DOCS_DIR} && npm run docusaurus docs:version ${version}`)

console.log(`Moving snapshot`)
fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_DOCS_FOLDER, VERSIONED_DOCS_FOLDER, {recursive: true})
fs.cpSync(REPO_DOCS_DIR+"/"+VERSIONED_SIDEBARS_FOLDER, VERSIONED_SIDEBARS_FOLDER, {recursive: true})

}

fs.cpSync(REPO_DOCS_DIR+"/versions.json", "versions.json")

0 comments on commit cb9c44c

Please sign in to comment.