Support documentation packages for Airflow 2.0 (#318)

docs-archive/apache-airflow-providers-google/0.0.1/index.html

@@ -0,0 +1 @@
+Placeholder for apache-airflow-providers-google v0.0.1

docs-archive/apache-airflow-providers-google/0.0.2/index.html

@@ -0,0 +1 @@
+Placeholder for apache-airflow-providers-google v0.0.2

docs-archive/apache-airflow-providers-google/stable.txt

@@ -0,0 +1 @@
+0.0.2

docs-archive/apache-airflow-providers/index.html

@@ -0,0 +1 @@
+Placeholder for apache-airflow-providers

dump-docs-packages-metadata.py

@@ -30,7 +30,10 @@ def dump_docs_package_metadata():
             "all-versions": get_all_versions(os.path.join(APACHE_AIRFLOW_ARCHIVE, package_name)),
         }
         for package_name in os.listdir(APACHE_AIRFLOW_ARCHIVE)
-        if not package_name.startswith(".")  # Exclude .DS_Store/
+        if (
+            not package_name.startswith(".") and  # Exclude .DS_Store/
+            os.path.isfile(os.path.join(os.path.join(APACHE_AIRFLOW_ARCHIVE, package_name, 'stable.txt')))
+        )
     ]
 
     json.dump(all_packages_infos, sys.stdout, indent=2)

landing-pages/site/content/en/docs/_index.md

@@ -0,0 +1,155 @@
+---
+title: "Documentation"
+linkTitle: "Documentation"
+weight: 20
+menu:
+    main:
+        weight: 10
+---
+
+# Documentation
+
+## [Apache Airflow](/docs/apache-airflow/stable/index.html)
+
+Apache Airflow Core, which includes webserver, scheduler, CLI and other components that are needed for minimal Airflow installation.
+
+## [Providers packages](/docs/apache-airflow-providers/index.html)
+
+Providers packages include integrations with third party integrations. They are updated independently of the Apache Airflow core.
+
+<ul class="list-providers">
+
+    <li><a href="/docs/apache-airflow-providers-amazon/stable/index.html">Amazon</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-cassandra/stable/index.html">Apache Cassandra</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-druid/stable/index.html">Apache Druid</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-hdfs/stable/index.html">Apache HDFS</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-hive/stable/index.html">Apache Hive</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-kylin/stable/index.html">Apache Hive</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-livy/stable/index.html">Apache Livy</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-pig/stable/index.html">Apache Pig</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-pinot/stable/index.html">Apache Pinot</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-spark/stable/index.html">Apache Spark</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-apache-sqoop/stable/index.html">Apache Sqoop</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-celery/stable/index.html">Celery</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-cloudant/stable/index.html">IBM Cloudant</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-cncf-kubernetes/stable/index.html">Kubernetes</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-databricks/stable/index.html">Databricks</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-datadog/stable/index.html">Datadog</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-dingding/stable/index.html">Dingding</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-discord/stable/index.html">Discord</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-docker/stable/index.html">Docker</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-elasticsearch/stable/index.html">Elasticsearch</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-exasol/stable/index.html">Exasol</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-facebook/stable/index.html">Facebook</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-ftp/stable/index.html">File Transfer Protocol (FTP)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-google/stable/index.html">Google</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-grpc/stable/index.html">gRPC</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-hashicorp/stable/index.html">Hashicorp</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-http/stable/index.html">Hypertext Transfer Protocol (HTTP)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-imap/stable/index.html">Internet Message Access Protocol (IMAP)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-jdbc/stable/index.html">Java Database Connectivity (JDBC)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-jenkins/stable/index.html">Jenkins</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-jira/stable/index.html">Jira</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-microsoft-azure/stable/index.html">Microsoft Azure</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-microsoft-mssql/stable/index.html">Microsoft SQL Server (MSSQL)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-microsoft-winrm/stable/index.html">Windows Remote Management (WinRM)</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-mongo/stable/index.html">MongoDB</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-mysql/stable/index.html">MySQL</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-odbc/stable/index.html">ODBC</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-openfaas/stable/index.html">OpenFaaS</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-opsgenie/stable/index.html">Opsgenie</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-oracle/stable/index.html">Orcle</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-pagerduty/stable/index.html">Pagerduty</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-papermill/stable/index.html">Papermill</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-plexus/stable/index.html">Plexus</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-postgres/stable/index.html">PostgreSQL</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-presto/stable/index.html">Presto</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-qubole/stable/index.html">Qubole</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-redis/stable/index.html">Redis</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-salesforce/stable/index.html">Salesforce</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-samba/stable/index.html">Samba</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-segment/stable/index.html">Segment</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-sendgrid/stable/index.html">Sendgrid</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-sftp/stable/index.html">SFTP</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-singularity/stable/index.html">Singularity</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-slack/stable/index.html">Slack</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-snowflake/stable/index.html">Snowflake</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-sqlite/stable/index.html">SQLite</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-ssh/stable/index.html">SSH</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-vertica/stable/index.html">Vertica</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-yandex/stable/index.html">Yandex</a></li>
+
+    <li><a href="/docs/apache-airflow-providers-zendesk/stable/index.html">Zendesk</a></li>
+
+</ul>
+
+<style>
+    @media (min-width: 768px) {
+        .list-providers {
+            column-count: 2
+        }
+    }
+    @media (min-width: 1000px) {
+        .list-providers {
+            column-count: 3
+        }
+    }
+</style>

landing-pages/site/layouts/docs/baseof.html

@@ -0,0 +1,44 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+<!doctype html>
+<html lang="{{ .Site.Language.Lang }}" class="no-js">
+<head>
+    {{ partial "head.html" . }}
+</head>
+<body class="td-{{ .Kind }}">
+<header>
+    {{ partial "navbar.html" . }}
+</header>
+<div class="roadmap container-fluid td-default base-layout">
+    <div class="d-flex">
+        <main class="col-12 col-md-9 col-xl-8 mx-auto" role="main">
+            {{ block "main" . }}{{ end }}
+        </main>
+    </div>
+    {{ partial "suggest-change" . }}
+</div>
+{{ partialCached "footer.html" . }}
+{{ partialCached "scripts.html" . }}
+</body>
+{{ $script := .Site.Data.webpack.main }}
+{{ with $script.js }}
+    <script src="{{ relURL . }}"></script>
+{{ end }}
+</html>

landing-pages/site/layouts/docs/list.html

@@ -0,0 +1,24 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+{{ define "main" }}
+    <div class="rst-content">
+        {{ .Content }}
+    </div>
+{{ end }}

landing-pages/site/layouts/docs/single.html

@@ -0,0 +1,24 @@
+    {{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+{{ define "main" }}
+    <div class="rst-content">
+        {{ .Content }}
+    </div>
+{{ end }}

site.sh

@@ -263,23 +263,29 @@ function build_site {
     mkdir -p dist
     rm -rf dist/*
     verbose_copy landing-pages/dist/. dist/
-    rm -rf dist/docs/* || true
-    mkdir -p dist/docs/apache-airflow/
     for pkg_path in docs-archive/*/ ; do
+        # Process directories only,
+        if [ ! -d "${pkg_path}" ]; then
+            continue;
+        fi
+
         package_name="$(basename -- "${pkg_path}")"
-        for ver_path in "docs-archive/${package_name}"/*/ ; do
-            version="$(basename -- "${ver_path}")"
-            verbose_copy "docs-archive/${package_name}/${version}/." "dist/docs/${package_name}/${version}"
-        done
-        stable_version="$(cat "docs-archive/${package_name}/stable.txt")"
-        verbose_copy "docs-archive/${package_name}/${stable_version}/." "dist/docs/${package_name}/stable"
-        create_redirect "dist/docs/${package_name}/index.html" "/docs/${package_name}/stable/index.html"
-    done
-    # TODO(mik-laj): For Airflow 1.10, we have one package so we don't need a separate index.
-    #     For Airflow 2.0, we need a separate index, because we also have a provider packages.
-    create_redirect "dist/docs/index.html" "/docs/apache-airflow/stable/index.html"
 
-    # This file may already have been created when building landing pages,
+        # Is this documentation versioned?
+        if [ -f "${pkg_path}/stable.txt" ]; then
+            mkdir -p "docs-archive/${package_name}"
+            for ver_path in "docs-archive/${package_name}"/*/ ; do
+                version="$(basename -- "${ver_path}")"
+                verbose_copy "docs-archive/${package_name}/${version}/." "dist/docs/${package_name}/${version}"
+            done
+            stable_version="$(cat "docs-archive/${package_name}/stable.txt")"
+            verbose_copy "docs-archive/${package_name}/${stable_version}/." "dist/docs/${package_name}/stable"
+            create_redirect "dist/docs/${package_name}/index.html" "/docs/${package_name}/stable/index.html"
+        else
+            verbose_copy "docs-archive/${package_name}/." "dist/docs/${package_name}/"
+        fi
+    done
+    # This file may already have been created during building landing pages,
     # but when building a full site, it's worth regenerate
     log "Preparing packages-metadata.json"
     python dump-docs-packages-metadata.py > "dist/_gen/packages-metadata.json"