odpi · mandy-chessell · Oct 7, 2024 · Oct 7, 2024
diff --git a/compose-configs/egeria-platform-jupyter-compose/README.md b/compose-configs/egeria-platform-jupyter-compose/README.md
@@ -20,7 +20,7 @@ Please feel free to engage with the community on our slack channel - we'd love y
 
 For a quick and simple environment to explore some of Egeria's base capabilities, the **egeria-platform-jupyter.yaml**  Docker Compose
 deployment may be a good starting point. Once this script executes successfully, you will have three docker containers running. 
-One for the Egeria platform, one for Kafkaand one for Jupyter. With this running configuration, you can work with any of Egeria's standard interfaces 
+One for the Egeria platform, one for Kafka and one for Jupyter. With this running configuration, you can work with any of Egeria's standard interfaces 
 - java APIs, python APIs, or just plain RESTful http calls - and of course, to make use of tools and interfaces that have been built using these APIs.
 
 The set of **Docker Compose** configurations will grow and evolve over time to cover additional scenarios. For example,

diff --git a/exchange/loading-bay/secrets/default.omsecrets b/exchange/loading-bay/secrets/default.omsecrets
@@ -2,12 +2,11 @@
 # Copyright Contributors to the Egeria project.
 
 secretsCollections:
-    collectionName:
+    PostgreSQL Server:LocalPostgreSQL1:
          refreshTimeInterval: 10
          secrets:
-             secretProperty1: secretValue1
-             secretProperty2: secretValue2
-             secretProperty3: secretValue3
+             userId: egeria_user
+             clearPassword: user4egeria
     View Server:view-server[b5423a9c-24d1-4d90-9bc1-5ae358eb7826]:
          refreshTimeInterval: 60
          tokenAPI:

diff --git a/...ataloguing-and-surveys/apache-kafka/images/open-lineage-listener-configured.png b/...ataloguing-and-surveys/apache-kafka/images/open-lineage-listener-configured.png
diff --git a/workspaces/cataloguing-and-surveys/apache-kafka/kafka-open-lineage-events.ipynb b/workspaces/cataloguing-and-surveys/apache-kafka/kafka-open-lineage-events.ipynb
@@ -1,30 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "f7cb0c66-1506-46d8-9a7e-031379fbbe14",
-   "metadata": {
-    "editable": true,
-    "slideshow": {
-     "slide_type": ""
-    },
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "view_server = os.environ.get(\"VIEW_SERVER\",\"view-server\")\n",
-    "url = os.environ.get(\"EGERIA_VIEW_SERVER_URL\",\"https://host.docker.internal:9443\")\n",
-    "user_id = os.environ.get(\"EGERIA_USER\", \"peterprofile\")\n",
-    "user_pwd = os.environ.get(\"EGERIA_USER_PASSWORD\")\n",
-    "\n",
-    "from pyegeria import EgeriaTech, TEMPLATE_GUIDS, INTEGRATION_GUIDS\n",
-    "import asyncio\n",
-    "import nest_asyncio\n",
-    "nest_asyncio.apply()\n"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "5d3ce06d-0a7d-4fe1-bb5f-4aa56623a884",
@@ -48,17 +23,52 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "id": "dfe33057-8389-4394-93db-a2aa3c3d3522",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize pyegeria\n",
+    "\n",
+    "import os\n",
+    "view_server = os.environ.get(\"VIEW_SERVER\",\"view-server\")\n",
+    "url = os.environ.get(\"EGERIA_VIEW_SERVER_URL\",\"https://host.docker.internal:9443\")\n",
+    "user_id = os.environ.get(\"EGERIA_USER\", \"peterprofile\")\n",
+    "user_pwd = os.environ.get(\"EGERIA_USER_PASSWORD\")\n",
+    "\n",
+    "from pyegeria import EgeriaTech, TEMPLATE_GUIDS, INTEGRATION_GUIDS\n",
+    "import asyncio\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
    "id": "88a3e1fc-683c-4ec7-95bd-331f8e841c6b",
    "metadata": {},
    "outputs": [],
    "source": [
+    "# Create client to egeria\n",
+    "\n",
     "egeria_client = EgeriaTech(view_server, url, user_id, user_pwd)\n",
     "token = egeria_client.create_egeria_bearer_token()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "d2712c19-fe40-4d43-96df-2e4edeff632b",
+   "metadata": {},
+   "source": [
+    "-----\n",
+    "\n",
+    "We begin by defining which Apache Kafka topic is being used by other systems to publish open lineage events.\n",
+    "\n",
+    "----"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 6,
    "id": "946b87f0-596c-463b-b180-ebb3c2bcab75",
    "metadata": {
     "editable": true,
@@ -72,7 +82,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "GUID of Kafka Topic is: 7c4626e8-3d0f-4a6c-b918-ff8d99e7f675\n"
+      "GUID of Kafka Topic is: 637f94b5-29b3-483f-b56c-adaaeab50090\n"
      ]
     }
    ],
@@ -84,8 +94,8 @@
     "    \"isOwnAnchor\": True,\n",
     "    \"placeholderPropertyValues\": {\n",
     "        \"serverName\" : \"LocalKafka\",\n",
-    "        \"hostIdentifier\": \"localhost\",\n",
-    "        \"portNumber\": \"9092\",\n",
+    "        \"hostIdentifier\": \"host.docker.internal\",\n",
+    "        \"portNumber\": \"9192\",\n",
     "        \"description\" : \"Receive open lineage events via Kafka Topic.\",\n",
     "        \"topicDescription\": \"Open Lineage Events for Backend Proxy\",\n",
     "        \"fullTopicName\": \"openlineage.events\",\n",
@@ -118,7 +128,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 7,
    "id": "1c9ddeb8-e693-4c85-a637-b6081feae3df",
    "metadata": {
     "editable": true,
@@ -131,10 +141,10 @@
     {
      "data": {
       "text/plain": [
-       "'c5321f92-3e76-43a7-834e-daefa85b3fbb'"
+       "'8f376c9b-2c11-49d1-85fe-ebb834f7e76a'"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -145,18 +155,34 @@
     "\n",
     "egeria_client.add_catalog_target(kafka_open_lineage_listener_guid, \n",
     "                                 endpoint_guid, \n",
-    "                                 \"localOpenLineage\",\n",
+    "                                 \"incomingOpenLineage\",\n",
     "                                 None,\n",
     "                                 None,\n",
     "                                 None,\n",
     "                                 None)\n",
     "                      \n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "567bff4d-db4b-46b9-b0c7-015758663339",
+   "metadata": {},
+   "source": [
+    "----\n",
+    "\n",
+    "You can see that the **OpenLineageKafkaListener** is configured to listen on the topic by using the `hey_egeria_ops show integrations status` command in a Terminal window of this JupyterLab environment.\n",
+    "\n",
+    "![Open Lineage Listener Configured](images/open-lineage-listener-configured.png)\n",
+    "\n",
+    "\n",
+    "## Testing the integration\n",
+    "\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "442cf0b2-e76a-4bc8-a5d9-2ecfe2e73b2c",
+   "id": "0705fedf-0f30-42f0-9a4d-9513d528372a",
    "metadata": {},
    "outputs": [],
    "source": []

diff --git a/workspaces/cataloguing-and-surveys/postgres/survey-and-catalog-postgres.ipynb b/workspaces/cataloguing-and-surveys/postgres/survey-and-catalog-postgres.ipynb
@@ -0,0 +1,173 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bbf73747-32b1-4088-9720-b5dcb7d5bbd1",
+   "metadata": {},
+   "source": [
+    "<!-- SPDX-License-Identifier: CC-BY-4.0 -->\n",
+    "<!-- Copyright Contributors to the ODPi Egeria project 2024. -->\n",
+    "\n",
+    "![Egeria Logo](https://raw.githubusercontent.com/odpi/egeria/main/assets/img/ODPi_Egeria_Logo_color.png)\n",
+    "\n",
+    "### Egeria Workbook\n",
+    "\n",
+    "# Cataloguing and surveying PostgreSQL Servers\n",
+    "\n",
+    "## Introduction"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "448b3b48-1ecf-49da-9445-117fdd9e24fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "view_server = os.environ.get(\"VIEW_SERVER\",\"view-server\")\n",
+    "url = os.environ.get(\"EGERIA_VIEW_SERVER_URL\",\"https://localhost:9443\")\n",
+    "user_id = os.environ.get(\"EGERIA_USER\", \"peterprofile\")\n",
+    "user_pwd = os.environ.get(\"EGERIA_USER_PASSWORD\")\n",
+    "\n",
+    "from pyegeria import EgeriaTech\n",
+    "import asyncio\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a4edcd1b-3177-435b-a631-6380d761eaf8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "egeria_tech = EgeriaTech(view_server, url, user_id, user_pwd)\n",
+    "token = egeria_tech.create_egeria_bearer_token()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "90e4f374-25e5-4a2d-8eb4-62ae8c4baa4f",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "\n",
+    "## Loading support for PostgreSQL Servers\n",
+    "\n",
+    "The definition of the postgres connectors, templates and associated reference data are loaded via a [Content Pack](https://egeria-project.org/content-packs/) called `PostgresContentPack.omarchive`.  The content pack can be loaded multiple times without ill-effect so run the following command to make sure it is loaded.\n",
+    "\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe30e926-e414-4a44-bab4-f6ddd32f64f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "egeria_tech.add_archive_file(\"content-packs/PostgresContentPack.omarchive\", None, \"active-metadata-store\")\n",
+    "\n",
+    "print(\"Archive loaded!\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c540da1f-2da5-4b19-aedb-4191f08462f4",
+   "metadata": {},
+   "source": [
+    "----\n",
+    "\n",
+    "## Survey a PostgreSQL Server\n",
+    "\n",
+    "Egeria's PostgreSQL support includes the ability to survey the contents of a PostgreSQL Server to discover the databases that is manages.  This command creates a description of the PostgreSQL Server and runs a survey to understand its contents.  A summary of the survey results can be found in /distribution-hub/surveys.\n",
+    "\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1204b07f-d405-4783-b4ef-0df90c010609",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "createAndSurveyServerName=\"PostgreSQLServer:CreateAndSurveyGovernanceActionProcess\"\n",
+    "\n",
+    "requestParameters = {\n",
+    "    \"serverName\" : \"LocalPostgreSQL1\",\n",
+    "    \"hostIdentifier\" : \"host.docker.internal\",\n",
+    "    \"portNumber\" : \"5442\",\n",
+    "    \"secretsStorePathName\" : \"loading-bay/secrets/default.omsecrets\",\n",
+    "    \"versionIdentifier\" : \"1.0\",\n",
+    "    \"description\" : \"PostgreSQL database in egeria-workspaces.\"\n",
+    "}\n",
+    "\n",
+    "egeria_tech.initiate_gov_action_process(createAndSurveyServerName, None, None, None, requestParameters, None, None)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e0d0f33e-165d-4289-b268-3d711581fd94",
+   "metadata": {},
+   "source": [
+    "----\n",
+    "\n",
+    "This next command catalogs the databases it finds on the PostgreSQL Server.\n",
+    "\n",
+    "----"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e1154b5-add3-4cb6-843c-5ea0de20c3e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "createAndCatalogServerName=\"PostgreSQLServer:CreateAndCatalogGovernanceActionProcess\"\n",
+    "\n",
+    "requestParameters = {\n",
+    "    \"serverName\" : \"LocalPostgreSQL1\",\n",
+    "    \"hostIdentifier\" : \"host.docker.internal\",\n",
+    "    \"portNumber\" : \"5442\",\n",
+    "    \"secretsStorePathName\" : \"loading-bay/secrets/default.omsecrets\",\n",
+    "    \"versionIdentifier\" : \"1.0\",\n",
+    "    \"description\" : \"PostgreSQL database in egeria-workspaces.\"\n",
+    "}\n",
+    "\n",
+    "egeria_tech.initiate_gov_action_process(createAndCatalogServerName, None, None, None, requestParameters, None, None)\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}