From 5e4ca6c5d0c1dbdddde7bb9432508c36c72cc095 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 19 Sep 2022 23:47:19 +0100 Subject: [PATCH] feat(docs): Adds docs on K8s scheduled ingestion --- docs-website/sidebars.js | 1 + .../schedule_docs/kubernetes.md | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 metadata-ingestion/schedule_docs/kubernetes.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 635f43dfabd338..0fce4f5aca3a01 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -150,6 +150,7 @@ module.exports = { "metadata-ingestion/schedule_docs/intro", "metadata-ingestion/schedule_docs/cron", "metadata-ingestion/schedule_docs/airflow", + "metadata-ingestion/schedule_docs/kubernetes", ], }, // { diff --git a/metadata-ingestion/schedule_docs/kubernetes.md b/metadata-ingestion/schedule_docs/kubernetes.md new file mode 100644 index 00000000000000..ace98cc354ca9a --- /dev/null +++ b/metadata-ingestion/schedule_docs/kubernetes.md @@ -0,0 +1,47 @@ +# Using Kubernetes + +If you have deployed DataHub using our official [helm charts](https://github.com/acryldata/datahub-helm) you can use the +datahub ingestion cron subchart to schedule ingestions. + +Here is an example of what that configuration would look like in your **values.yaml**: + +```yaml +datahub-ingestion-cron: + enabled: true + crons: + mysql: + schedule: "0 * * * *" # Every hour + recipe: + configmapName: recipe-config + fileName: mysql_recipe.yml +``` + +This assumes the pre-existence of a Kubernetes ConfigMap which holds all recipes being scheduled in the same namespace as +where the cron jobs will be running. + +An example could be: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: recipe-config +data: + mysql_recipe.yml: |- + source: + type: mysql + config: + # Coordinates + host_port: :3306 + database: dbname + + # Credentials + username: root + password: example + + sink: + type: datahub-rest + config: + server: http://:8080 +``` + +For more information, please see the [documentation](https://github.com/acryldata/datahub-helm/tree/master/charts/datahub/subcharts/datahub-ingestion-cron) of this sub-chart. \ No newline at end of file