Merge branch 'main' of https://github.com/open-metadata/OpenMetadata …

…into feature/cassandra-ssl
open-metadata · Jan 6, 2025 · 0e0b91e · 0e0b91e
2 parents 95bbff3 + 9adb507
commit 0e0b91e
Show file tree

Hide file tree

Showing 22 changed files with 89 additions and 36 deletions.
diff --git a/ingestion/src/metadata/profiler/orm/functions/length.py b/ingestion/src/metadata/profiler/orm/functions/length.py
@@ -43,6 +43,7 @@ def _(element, compiler, **kw):
 @compiles(LenFn, Dialects.MariaDB)
 @compiles(LenFn, Dialects.Athena)
 @compiles(LenFn, Dialects.Trino)
+@compiles(LenFn, Dialects.PinotDB)
 @compiles(LenFn, Dialects.Presto)
 @compiles(LenFn, Dialects.BigQuery)
 @compiles(LenFn, Dialects.Oracle)

diff --git a/ingestion/src/metadata/profiler/orm/registry.py b/ingestion/src/metadata/profiler/orm/registry.py
@@ -76,6 +76,7 @@ class PythonDialects(Enum):
     MSSQL = "mssql"
     MySQL = "mysql"
     Oracle = "oracle"
+    PinotDB = "pinotdb"
     Postgres = "postgresql"
     Presto = "presto"
     Redshift = "redshift"

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/mariadb/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/mariadb/yaml.md
@@ -16,6 +16,7 @@ In this section, we provide guides and references to use the MariaDB connector.
 Configure and schedule MariaDB metadata and profiler workflows from the OpenMetadata UI:
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -155,6 +156,8 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "mariadb"} /%}
+
 {% partial file="/v1.6/connectors/yaml/data-profiler.md" variables={connector: "mariadb"} /%}
 
 {% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "mariadb"} /%}

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md
@@ -18,6 +18,7 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](#data-profiler)
+- [Lineage](#lineage)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
 - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata)
@@ -476,6 +477,8 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "mysql"} /%}
+
 {% partial file="/v1.6/connectors/yaml/data-profiler.md" variables={connector: "mysql"} /%}
 
 {% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "mysql"} /%}

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/oracle/yaml.md
@@ -237,9 +237,7 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/data-quality.md" /%}
 
-## Lineage
-
-You can learn more about how to ingest lineage [here](/connectors/ingestion/workflows/lineage).
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "oracle"} /%}
 
 ## dbt Integration
 

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/pinotdb/index.md b/openmetadata-docs/content/v1.6.x/connectors/database/pinotdb/index.md
@@ -19,6 +19,7 @@ Configure and schedule PinotDB metadata and profiler workflows from the OpenMeta
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 
 {% partial file="/v1.6/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/pinotdb/yaml"} /%}
 

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/pinotdb/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/pinotdb/yaml.md
@@ -17,6 +17,7 @@ Configure and schedule PinotDB metadata and profiler workflows from the OpenMeta
 
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -149,6 +150,8 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "pinotdb"} /%}
+
 {% partial file="/v1.6/connectors/yaml/data-profiler.md" variables={connector: "pinotdb"} /%}
 
 {% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "pinotdb"} /%}

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/sap-hana/index.md b/openmetadata-docs/content/v1.6.x/connectors/database/sap-hana/index.md
@@ -19,6 +19,7 @@ Configure and schedule SAP Hana metadata and profiler workflows from the OpenMet
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
 

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/sap-hana/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/sap-hana/yaml.md
@@ -17,6 +17,7 @@ Configure and schedule SAP Hana metadata and profiler workflows from the OpenMet
 
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -221,6 +222,8 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "sapHana"} /%}
+
 {% partial file="/v1.6/connectors/yaml/data-profiler.md" variables={connector: "sapHana"} /%}
 
 {% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "sapHana"} /%}

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/singlestore/index.md b/openmetadata-docs/content/v1.6.x/connectors/database/singlestore/index.md
@@ -18,6 +18,7 @@ Configure and schedule Singlestore metadata and profiler workflows from the Open
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
 

diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/singlestore/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/singlestore/yaml.md
@@ -19,6 +19,7 @@ Configure and schedule Singlestore metadata and profiler workflows from the Open
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](#data-profiler)
+- [Lineage](#lineage)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
 
@@ -151,6 +152,8 @@ source:
 
 {% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.6/connectors/yaml/lineage.md" variables={connector: "singlestore"} /%}
+
 {% partial file="/v1.6/connectors/yaml/data-profiler.md" variables={connector: "singlestore"} /%}
 
 {% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "singlestore"} /%}

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mariadb/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mariadb/yaml.md
@@ -16,6 +16,7 @@ In this section, we provide guides and references to use the MariaDB connector.
 Configure and schedule MariaDB metadata and profiler workflows from the OpenMetadata UI:
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -155,6 +156,8 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "mariadb"} /%}
+
 {% partial file="/v1.7/connectors/yaml/data-profiler.md" variables={connector: "mariadb"} /%}
 
 {% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "mariadb"} /%}

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md
@@ -18,6 +18,7 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](#data-profiler)
+- [Lineage](#lineage)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
 - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata)
@@ -476,6 +477,8 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "mysql"} /%}
+
 {% partial file="/v1.7/connectors/yaml/data-profiler.md" variables={connector: "mysql"} /%}
 
 {% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "mysql"} /%}

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md
@@ -237,9 +237,7 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/data-quality.md" /%}
 
-## Lineage
-
-You can learn more about how to ingest lineage [here](/connectors/ingestion/workflows/lineage).
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "oracle"} /%}
 
 ## dbt Integration
 

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/pinotdb/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/pinotdb/index.md
@@ -19,6 +19,7 @@ Configure and schedule PinotDB metadata and profiler workflows from the OpenMeta
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 
 {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/pinotdb/yaml"} /%}
 

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/pinotdb/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/pinotdb/yaml.md
@@ -17,6 +17,7 @@ Configure and schedule PinotDB metadata and profiler workflows from the OpenMeta
 
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -149,6 +150,8 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "pinotdb"} /%}
+
 {% partial file="/v1.7/connectors/yaml/data-profiler.md" variables={connector: "pinotdb"} /%}
 
 {% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "pinotdb"} /%}

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/sap-hana/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/sap-hana/index.md
@@ -19,6 +19,7 @@ Configure and schedule SAP Hana metadata and profiler workflows from the OpenMet
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
 

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/sap-hana/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/sap-hana/yaml.md
@@ -17,6 +17,7 @@ Configure and schedule SAP Hana metadata and profiler workflows from the OpenMet
 
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
+- [Lineage](#lineage)
 - [Data Profiler](#data-profiler)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
@@ -221,6 +222,8 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "sapHana"} /%}
+
 {% partial file="/v1.7/connectors/yaml/data-profiler.md" variables={connector: "sapHana"} /%}
 
 {% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "sapHana"} /%}

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/singlestore/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/singlestore/index.md
@@ -18,6 +18,7 @@ Configure and schedule Singlestore metadata and profiler workflows from the Open
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](/how-to-guides/data-quality-observability/profiler/workflow)
+- [Lineage](/how-to-guides/data-lineage/workflow)
 - [Data Quality](/how-to-guides/data-quality-observability/quality)
 - [dbt Integration](/connectors/ingestion/workflows/dbt)
 

diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/singlestore/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/singlestore/yaml.md
@@ -19,6 +19,7 @@ Configure and schedule Singlestore metadata and profiler workflows from the Open
 - [Requirements](#requirements)
 - [Metadata Ingestion](#metadata-ingestion)
 - [Data Profiler](#data-profiler)
+- [Lineage](#lineage)
 - [Data Quality](#data-quality)
 - [dbt Integration](#dbt-integration)
 
@@ -151,6 +152,8 @@ source:
 
 {% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}
 
+{% partial file="/v1.7/connectors/yaml/lineage.md" variables={connector: "singlestore"} /%}
+
 {% partial file="/v1.7/connectors/yaml/data-profiler.md" variables={connector: "singlestore"} /%}
 
 {% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "singlestore"} /%}

diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/auth.setup.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/auth.setup.ts
@@ -10,43 +10,16 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-import { Page, test as setup } from '@playwright/test';
-import { JWT_EXPIRY_TIME_MAP } from '../constant/login';
+import { test as setup } from '@playwright/test';
 import { AdminClass } from '../support/user/AdminClass';
-import { getApiContext } from '../utils/common';
-import { updateJWTTokenExpiryTime } from '../utils/login';
-import {
-  updateDefaultDataConsumerPolicy,
-  updateDefaultOrganizationPolicy,
-} from '../utils/permission';
-import { removeOrganizationPolicyAndRole } from '../utils/team';
+import { loginAsAdmin } from '../utils/initialSetup';
 const adminFile = 'playwright/.auth/admin.json';
 
-const initialSetup = async (page: Page) => {
-  const { apiContext, afterAction } = await getApiContext(page);
-  // Update JWT expiry time to 4 hours
-  await updateJWTTokenExpiryTime(apiContext, JWT_EXPIRY_TIME_MAP['4 hours']);
-  // Remove organization policy and role
-  await removeOrganizationPolicyAndRole(apiContext);
-  // update default Organization policy
-  await updateDefaultOrganizationPolicy(apiContext);
-  // update default Data consumer policy
-  await updateDefaultDataConsumerPolicy(apiContext);
-
-  await afterAction();
-};
-
 setup('authenticate as admin', async ({ page }) => {
   const admin = new AdminClass();
 
   // login with admin user
-  await admin.login(page);
-  await page.waitForURL('**/my-data');
-  await initialSetup(page);
-  await admin.logout(page);
-  await page.waitForURL('**/signin');
-  await admin.login(page);
-  await page.waitForURL('**/my-data');
+  await loginAsAdmin(page, admin);
 
   // End of authentication steps.
   await page.context().storageState({ path: adminFile });

diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/initialSetup.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/initialSetup.ts
@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { Page } from '@playwright/test';
+import { JWT_EXPIRY_TIME_MAP } from '../constant/login';
+import { AdminClass } from '../support/user/AdminClass';
+import { getApiContext } from './common';
+import { updateJWTTokenExpiryTime } from './login';
+import {
+  updateDefaultDataConsumerPolicy,
+  updateDefaultOrganizationPolicy,
+} from './permission';
+import { removeOrganizationPolicyAndRole } from './team';
+
+const initialSetup = async (page: Page) => {
+  const { apiContext, afterAction } = await getApiContext(page);
+  // Update JWT expiry time to 4 hours
+  await updateJWTTokenExpiryTime(apiContext, JWT_EXPIRY_TIME_MAP['4 hours']);
+  // Remove organization policy and role
+  await removeOrganizationPolicyAndRole(apiContext);
+  // update default Organization policy
+  await updateDefaultOrganizationPolicy(apiContext);
+  // update default Data consumer policy
+  await updateDefaultDataConsumerPolicy(apiContext);
+
+  await afterAction();
+};
+
+export const loginAsAdmin = async (page: Page, admin: AdminClass) => {
+  await admin.login(page);
+  await page.waitForURL('**/my-data');
+  await initialSetup(page);
+  await admin.logout(page);
+  await page.waitForURL('**/signin');
+  await admin.login(page);
+  await page.waitForURL('**/my-data');
+};