diff --git a/datacatalog/__init__.py b/datacatalog/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/quickstart/__init__.py b/datacatalog/quickstart/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/quickstart/conftest.py b/datacatalog/quickstart/conftest.py new file mode 100644 index 000000000000..c7657fb7b62a --- /dev/null +++ b/datacatalog/quickstart/conftest.py @@ -0,0 +1,78 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import uuid + +import google.auth +from google.cloud import bigquery +from google.cloud import datacatalog_v1 + +import pytest + + +def temp_suffix(): + now = datetime.datetime.now() + return "{}_{}".format( + now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] + ) + + +@pytest.fixture(scope="session") +def client(credentials): + return datacatalog_v1.DataCatalogClient(credentials=credentials) + + +@pytest.fixture(scope="session") +def bigquery_client(credentials, project_id): + return bigquery.Client(project=project_id, credentials=credentials) + + +@pytest.fixture(scope="session") +def default_credentials(): + return google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) + + +@pytest.fixture(scope="session") +def credentials(default_credentials): + return default_credentials[0] + + +@pytest.fixture(scope="session") +def project_id(default_credentials): + return default_credentials[1] + + +@pytest.fixture +def dataset_id(bigquery_client): + dataset_id = f"python_data_catalog_sample_{temp_suffix()}" + dataset = bigquery_client.create_dataset(dataset_id) + yield dataset.dataset_id + bigquery_client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) + + +@pytest.fixture +def table_id(bigquery_client, project_id, dataset_id): + table_id = f"python_data_catalog_sample_{temp_suffix()}" + table = bigquery.Table("{}.{}.{}".format(project_id, dataset_id, table_id)) + table = bigquery_client.create_table(table) + yield table.table_id + bigquery_client.delete_table(table, not_found_ok=True) + + +@pytest.fixture +def random_tag_template_id(): + random_tag_template_id = f"python_sample_{temp_suffix()}" + yield random_tag_template_id diff --git a/datacatalog/quickstart/create_fileset_entry_quickstart.py b/datacatalog/quickstart/create_fileset_entry_quickstart.py deleted file mode 100644 index 5e1c99f0f3d0..000000000000 --- a/datacatalog/quickstart/create_fileset_entry_quickstart.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def create_fileset_entry_quickstart(client, project_id, entry_group_id, entry_id): - - # [START datacatalog_create_fileset_quickstart_tag] - # Import required modules. - from google.cloud import datacatalog_v1beta1 - - # TODO(developer): Construct a Data Catalog client object. - # client = datacatalog_v1beta1.DataCatalogClient() - - # TODO(developer): Set project_id to your - # Google Cloud Platform project ID the entry will belong. - # project_id = "your-project-id" - - # TODO(developer): Specify the geographic location where the - # entry should reside. - # Currently, Data Catalog stores metadata in the us-central1 region. - location_id = "us-central1" - - # TODO(developer): Set entry_group_id to the ID of the entry group - # the entry will belong. - # entry_group_id = "your_entry_group_id" - - # TODO(developer): Set entry_id to the ID of the entry to create. - # entry_id = "your_entry_id" - - # Create an Entry Group. - # Construct a full Entry Group object to send to the API. - entry_group_obj = datacatalog_v1beta1.EntryGroup() - entry_group_obj.display_name = "My Fileset Entry Group" - entry_group_obj.description = "This Entry Group consists of ...." - - # Send the Entry Group to the API for creation. - # Raises google.api_core.exceptions.AlreadyExists if the Entry Group - # already exists within the project. - entry_group = client.create_entry_group( - request = {'parent': datacatalog_v1beta1.DataCatalogClient.location_path( - project_id, location_id - ), 'entry_group_id': entry_group_id, 'entry_group': entry_group_obj}) - print("Created entry group {}".format(entry_group.name)) - - # Create a Fileset Entry. - # Construct a full Entry object to send to the API. - entry = datacatalog_v1beta1.Entry() - entry.display_name = "My Fileset" - entry.description = "This Fileset consists of ..." - entry.gcs_fileset_spec.file_patterns.append("gs://cloud-samples-data/*") - entry.type = datacatalog_v1beta1.EntryType.FILESET - - # Create the Schema, for example when you have a csv file. - columns = [] - columns.append( - datacatalog_v1beta1.ColumnSchema( - column="first_name", - description="First name", - mode="REQUIRED", - type="STRING", - ) - ) - - columns.append( - datacatalog_v1beta1.ColumnSchema( - column="last_name", description="Last name", mode="REQUIRED", type="STRING" - ) - ) - - # Create sub columns for the addresses parent column - subcolumns = [] - subcolumns.append( - datacatalog_v1beta1.ColumnSchema( - column="city", description="City", mode="NULLABLE", type="STRING" - ) - ) - - subcolumns.append( - datacatalog_v1beta1.ColumnSchema( - column="state", description="State", mode="NULLABLE", type="STRING" - ) - ) - - columns.append( - datacatalog_v1beta1.ColumnSchema( - column="addresses", - description="Addresses", - mode="REPEATED", - subcolumns=subcolumns, - type="RECORD", - ) - ) - - entry.schema.columns.extend(columns) - - # Send the entry to the API for creation. - # Raises google.api_core.exceptions.AlreadyExists if the Entry already - # exists within the project. - entry = client.create_entry(request = {'parent': entry_group.name, 'entry_id': entry_id, 'entry': entry}) - print("Created entry {}".format(entry.name)) - # [END datacatalog_create_fileset_quickstart_tag] diff --git a/datacatalog/quickstart/noxfile.py b/datacatalog/quickstart/noxfile.py new file mode 100644 index 000000000000..5660f08be441 --- /dev/null +++ b/datacatalog/quickstart/noxfile.py @@ -0,0 +1,222 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/datacatalog/quickstart/quickstart.py b/datacatalog/quickstart/quickstart.py new file mode 100644 index 000000000000..f6579e53690f --- /dev/null +++ b/datacatalog/quickstart/quickstart.py @@ -0,0 +1,131 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def quickstart(override_values): + """Creates a tag template and attach a tag to a BigQuery table.""" + # [START data_catalog_quickstart] + # Import required modules. + from google.cloud import datacatalog_v1 + + # TODO: Set these values before running the sample. + # Google Cloud Platform project. + project_id = "my_project" + # Set dataset_id to the ID of existing dataset. + dataset_id = "demo_dataset" + # Set table_id to the ID of existing table. + table_id = "trips" + # Tag template to create. + tag_template_id = "example_tag_template" + + # [END data_catalog_quickstart] + + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + project_id = override_values.get("project_id", project_id) + dataset_id = override_values.get("dataset_id", dataset_id) + table_id = override_values.get("table_id", table_id) + tag_template_id = override_values.get("tag_template_id", tag_template_id) + + # [START data_catalog_quickstart] + # For all regions available, see: + # https://cloud.google.com/data-catalog/docs/concepts/regions + location = "us-central1" + + # Use Application Default Credentials to create a new + # Data Catalog client. GOOGLE_APPLICATION_CREDENTIALS + # environment variable must be set with the location + # of a service account key file. + datacatalog_client = datacatalog_v1.DataCatalogClient() + + # Create a Tag Template. + tag_template = datacatalog_v1.types.TagTemplate() + + tag_template.display_name = "Demo Tag Template" + + tag_template.fields["source"] = datacatalog_v1.types.TagTemplateField() + tag_template.fields["source"].display_name = "Source of data asset" + tag_template.fields[ + "source" + ].type_.primitive_type = datacatalog_v1.types.FieldType.PrimitiveType.STRING + + tag_template.fields["num_rows"] = datacatalog_v1.types.TagTemplateField() + tag_template.fields["num_rows"].display_name = "Number of rows in data asset" + tag_template.fields[ + "num_rows" + ].type_.primitive_type = datacatalog_v1.types.FieldType.PrimitiveType.DOUBLE + + tag_template.fields["has_pii"] = datacatalog_v1.types.TagTemplateField() + tag_template.fields["has_pii"].display_name = "Has PII" + tag_template.fields[ + "has_pii" + ].type_.primitive_type = datacatalog_v1.types.FieldType.PrimitiveType.BOOL + + tag_template.fields["pii_type"] = datacatalog_v1.types.TagTemplateField() + tag_template.fields["pii_type"].display_name = "PII type" + + for display_name in ["EMAIL", "SOCIAL SECURITY NUMBER", "NONE"]: + enum_value = datacatalog_v1.types.FieldType.EnumType.EnumValue( + display_name=display_name + ) + tag_template.fields["pii_type"].type_.enum_type.allowed_values.append( + enum_value + ) + + expected_template_name = datacatalog_v1.DataCatalogClient.tag_template_path( + project_id, location, tag_template_id + ) + + # Create the Tag Template. + try: + tag_template = datacatalog_client.create_tag_template( + parent=f"projects/{project_id}/locations/{location}", + tag_template_id=tag_template_id, + tag_template=tag_template, + ) + print(f"Created template: {tag_template.name}") + except OSError as e: + print(f"Cannot create template: {expected_template_name}") + print(f"{e}") + + # Lookup Data Catalog's Entry referring to the table. + resource_name = ( + f"//bigquery.googleapis.com/projects/{project_id}" + f"/datasets/{dataset_id}/tables/{table_id}" + ) + table_entry = datacatalog_client.lookup_entry( + request={"linked_resource": resource_name} + ) + + # Attach a Tag to the table. + tag = datacatalog_v1.types.Tag() + + tag.template = tag_template.name + tag.name = "my_super_cool_tag" + + tag.fields["source"] = datacatalog_v1.types.TagField() + tag.fields["source"].string_value = "Copied from tlc_yellow_trips_2018" + + tag.fields["num_rows"] = datacatalog_v1.types.TagField() + tag.fields["num_rows"].double_value = 113496874 + + tag.fields["has_pii"] = datacatalog_v1.types.TagField() + tag.fields["has_pii"].bool_value = False + + tag.fields["pii_type"] = datacatalog_v1.types.TagField() + tag.fields["pii_type"].enum_value.display_name = "NONE" + + tag = datacatalog_client.create_tag(parent=table_entry.name, tag=tag) + print(f"Created tag: {tag.name}") + # [END data_catalog_quickstart] diff --git a/datacatalog/quickstart/quickstart_test.py b/datacatalog/quickstart/quickstart_test.py new file mode 100644 index 000000000000..a63efee6cbf3 --- /dev/null +++ b/datacatalog/quickstart/quickstart_test.py @@ -0,0 +1,33 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import quickstart + + +def test_quickstart(capsys, client, project_id, dataset_id, table_id, random_tag_template_id): + location = "us-central1" + override_values = { + "project_id": project_id, + "dataset_id": dataset_id, + "table_id": table_id, + "tag_template_id": random_tag_template_id + } + tag_template_name = client.tag_template_path( + project_id, location, random_tag_template_id + ) + quickstart.quickstart(override_values) + out, err = capsys.readouterr() + assert "Created template: {}".format(tag_template_name) in out + assert "Created tag:" in out + client.delete_tag_template(name=tag_template_name, force=True) diff --git a/datacatalog/quickstart/requirements-test.txt b/datacatalog/quickstart/requirements-test.txt new file mode 100644 index 000000000000..55a7ce4c6db8 --- /dev/null +++ b/datacatalog/quickstart/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==6.2.4 +google-cloud-bigquery==2.4.0 \ No newline at end of file diff --git a/datacatalog/quickstart/requirements.txt b/datacatalog/quickstart/requirements.txt new file mode 100644 index 000000000000..5f898b7ae101 --- /dev/null +++ b/datacatalog/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-datacatalog==3.4.0 diff --git a/datacatalog/snippets/README.rst b/datacatalog/snippets/README.rst deleted file mode 100644 index 3476cceaf360..000000000000 --- a/datacatalog/snippets/README.rst +++ /dev/null @@ -1,139 +0,0 @@ - -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Data Catalog Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/README.rst - - -This directory contains samples for Google Cloud Data Catalog. `Google Cloud Data Catalog`_ is a fully managed and scalable metadata management service that empowers organizations to quickly discover, manage, and understand all their data in Google Cloud. - - - - -.. _Google Cloud Data Catalog: https://cloud.google.com/data-catalog/docs - - -Setup -------------------------------------------------------------------------------- - - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - - - - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 3.6+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - - - - - - -Samples -------------------------------------------------------------------------------- - - -Lookup entry -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/lookup_entry.py,datacatalog/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python lookup_entry.py - - - usage: lookup_entry.py [-h] - project_id - {bigquery-dataset,bigquery-table,pubsub-topic} ... - - This application demonstrates how to perform basic operations on entries - with the Cloud Data Catalog API. - - For more information, see the README.md under /datacatalog and the - documentation at https://cloud.google.com/data-catalog/docs. - - positional arguments: - project_id Your Google Cloud project ID - {bigquery-dataset,bigquery-table,pubsub-topic} - bigquery-dataset Retrieves Data Catalog entry for the given BigQuery - Dataset. - bigquery-table Retrieves Data Catalog entry for the given BigQuery - Table. - pubsub-topic Retrieves Data Catalog entry for the given Pub/Sub - Topic. - - optional arguments: - -h, --help show this help message and exit - - - - - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/datacatalog/snippets/README.rst.in b/datacatalog/snippets/README.rst.in deleted file mode 100644 index 704d55a5f9f0..000000000000 --- a/datacatalog/snippets/README.rst.in +++ /dev/null @@ -1,23 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Data Catalog - short_name: Data Catalog - url: https://cloud.google.com/data-catalog/docs - description: > - `Google Cloud Data Catalog`_ is a fully managed and scalable metadata - management service that empowers organizations to quickly discover, manage, - and understand all their data in Google Cloud. - -setup: -- auth -- install_deps - -samples: -- name: Lookup entry - file: lookup_entry.py - show_help: true - -cloud_client_library: true - -folder: datacatalog/cloud-client diff --git a/datacatalog/snippets/conftest.py b/datacatalog/snippets/conftest.py new file mode 100644 index 000000000000..089190d23042 --- /dev/null +++ b/datacatalog/snippets/conftest.py @@ -0,0 +1,148 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import uuid + +from google.api_core.exceptions import NotFound, PermissionDenied +import google.auth +from google.cloud import datacatalog_v1 + +import pytest + +datacatalog = datacatalog_v1.DataCatalogClient() + + +LOCATION = "us-central1" + + +def temp_suffix(): + now = datetime.datetime.now() + return "{}_{}".format(now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8]) + + +@pytest.fixture(scope="session") +def client(credentials): + return datacatalog_v1.DataCatalogClient(credentials=credentials) + + +@pytest.fixture(scope="session") +def default_credentials(): + return google.auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + +@pytest.fixture(scope="session") +def credentials(default_credentials): + return default_credentials[0] + + +@pytest.fixture(scope="session") +def project_id(default_credentials): + return default_credentials[1] + + +@pytest.fixture +def valid_member_id(client, project_id, random_existing_tag_template_id): + template_name = datacatalog_v1.DataCatalogClient.tag_template_path( + project_id, LOCATION, random_existing_tag_template_id + ) + + # Retrieve Template's current IAM Policy. + policy = datacatalog.get_iam_policy(resource=template_name) + yield policy.bindings[0].members[0] + + +@pytest.fixture +def resources_to_delete(client, project_id): + doomed = { + "entries": [], + "entry_groups": [], + "templates": [], + } + yield doomed + + for entry_name in doomed["entries"]: + try: + client.delete_entry(name=entry_name) + except (NotFound, PermissionDenied): + pass + for group_name in doomed["entry_groups"]: + try: + client.delete_entry_group(name=group_name) + except (NotFound, PermissionDenied): + pass + for template_name in doomed["templates"]: + try: + client.delete_tag_template(name=template_name, force=True) + except (NotFound, PermissionDenied): + pass + + +@pytest.fixture +def random_entry_id(): + random_entry_id = f"python_sample_entry_{temp_suffix()}" + yield random_entry_id + + +@pytest.fixture +def random_entry_group_id(): + random_entry_group_id = f"python_sample_group_{temp_suffix()}" + yield random_entry_group_id + + +@pytest.fixture +def random_tag_template_id(): + random_tag_template_id = f"python_sample_{temp_suffix()}" + yield random_tag_template_id + + +@pytest.fixture +def random_existing_tag_template_id(client, project_id, resources_to_delete): + random_tag_template_id = f"python_sample_{temp_suffix()}" + random_tag_template = datacatalog_v1.types.TagTemplate() + random_tag_template.fields["source"] = datacatalog_v1.types.TagTemplateField() + random_tag_template.fields[ + "source" + ].type_.primitive_type = datacatalog_v1.FieldType.PrimitiveType.STRING.value + random_tag_template = client.create_tag_template( + parent=datacatalog_v1.DataCatalogClient.common_location_path( + project_id, LOCATION + ), + tag_template_id=random_tag_template_id, + tag_template=random_tag_template, + ) + yield random_tag_template_id + resources_to_delete["templates"].append(random_tag_template.name) + + +@pytest.fixture +def random_existing_entry_group( + client, project_id, random_entry_group_id, resources_to_delete +): + entry_group_obj = datacatalog_v1.types.EntryGroup() + entry_group_obj.display_name = f"python_sample_{temp_suffix()}" + entry_group_obj.description = "Data Catalog samples test entry group." + + entry_group = datacatalog.create_entry_group( + parent=datacatalog_v1.DataCatalogClient.common_location_path( + project_id, LOCATION + ), + entry_group_id=random_entry_group_id, + entry_group=entry_group_obj, + ) + yield entry_group + resources_to_delete["entry_groups"].append(entry_group.name) diff --git a/datacatalog/snippets/create_custom_entry.py b/datacatalog/snippets/create_custom_entry.py new file mode 100644 index 000000000000..43a2dfac3c13 --- /dev/null +++ b/datacatalog/snippets/create_custom_entry.py @@ -0,0 +1,73 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_custom_entry(override_values): + """Creates a custom entry within an entry group.""" + # [START data_catalog_create_custom_entry] + # Import required modules. + from google.cloud import datacatalog_v1 + + # Google Cloud Platform project. + project_id = "my-project" + # Entry group to be created. + # For sample code demonstrating entry group creation, see quickstart: + # https://cloud.google.com/data-catalog/docs/quickstart-tagging + entry_group_name = "my_existing_entry_group" + # Entry to be created. + entry_id = "my_new_entry_id" + + # [END data_catalog_create_custom_entry] + + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + project_id = override_values.get("project_id", project_id) + entry_group_name = override_values.get("entry_group_name", entry_group_name) + entry_id = override_values.get("entry_id", entry_id) + + # [START data_catalog_create_custom_entry] + datacatalog = datacatalog_v1.DataCatalogClient() + + # Create an Entry. + entry = datacatalog_v1.types.Entry() + entry.user_specified_system = "onprem_data_system" + entry.user_specified_type = "onprem_data_asset" + entry.display_name = "My awesome data asset" + entry.description = "This data asset is managed by an external system." + entry.linked_resource = "//my-onprem-server.com/dataAssets/my-awesome-data-asset" + + # Create the Schema, this is optional. + entry.schema.columns.append( + datacatalog_v1.types.ColumnSchema( + column="first_column", + type_="STRING", + description="This columns consists of ....", + mode=None, + ) + ) + + entry.schema.columns.append( + datacatalog_v1.types.ColumnSchema( + column="second_column", + type_="DOUBLE", + description="This columns consists of ....", + mode=None, + ) + ) + + entry = datacatalog.create_entry( + parent=entry_group_name, entry_id=entry_id, entry=entry + ) + print("Created entry: {}".format(entry.name)) + # [END data_catalog_create_custom_entry] diff --git a/datacatalog/snippets/create_custom_entry_test.py b/datacatalog/snippets/create_custom_entry_test.py new file mode 100644 index 000000000000..742993eaa534 --- /dev/null +++ b/datacatalog/snippets/create_custom_entry_test.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import create_custom_entry + + +def test_create_custom_entry( + capsys, + client, + project_id, + random_existing_entry_group, + random_entry_group_id, + random_entry_id, + resources_to_delete, +): + location = "us-central1" + override_values = { + "project_id": project_id, + "entry_group_name": random_existing_entry_group.name, + "entry_id": random_entry_id, + } + expected_entry = client.entry_path( + project_id, location, random_entry_group_id, random_entry_id + ) + create_custom_entry.create_custom_entry(override_values) + out, err = capsys.readouterr() + assert f"Created entry: {expected_entry}" in out + resources_to_delete["entries"].append(expected_entry) diff --git a/datacatalog/snippets/create_fileset.py b/datacatalog/snippets/create_fileset.py new file mode 100644 index 000000000000..b76d96a611b1 --- /dev/null +++ b/datacatalog/snippets/create_fileset.py @@ -0,0 +1,105 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_fileset(override_values): + """Creates a fileset within an entry group.""" + # [START data_catalog_create_fileset] + # Import required modules. + from google.cloud import datacatalog_v1 + + # TODO: Set these values before running the sample. + project_id = "project_id" + fileset_entry_group_id = "entry_group_id" + fileset_entry_id = "entry_id" + + # [END data_catalog_create_fileset] + + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + project_id = override_values.get("project_id", project_id) + fileset_entry_group_id = override_values.get( + "fileset_entry_group_id", fileset_entry_group_id + ) + fileset_entry_id = override_values.get("fileset_entry_id", fileset_entry_id) + + # [START data_catalog_create_fileset] + # For all regions available, see: + # https://cloud.google.com/data-catalog/docs/concepts/regions + location = "us-central1" + + datacatalog = datacatalog_v1.DataCatalogClient() + + # Create an Entry Group. + entry_group_obj = datacatalog_v1.types.EntryGroup() + entry_group_obj.display_name = "My Fileset Entry Group" + entry_group_obj.description = "This Entry Group consists of ...." + + entry_group = datacatalog.create_entry_group( + parent=datacatalog_v1.DataCatalogClient.common_location_path( + project_id, location + ), + entry_group_id=fileset_entry_group_id, + entry_group=entry_group_obj, + ) + print(f"Created entry group: {entry_group.name}") + + # Create a Fileset Entry. + entry = datacatalog_v1.types.Entry() + entry.display_name = "My Fileset" + entry.description = "This fileset consists of ...." + entry.gcs_fileset_spec.file_patterns.append("gs://my_bucket/*.csv") + entry.type_ = datacatalog_v1.EntryType.FILESET + + # Create the Schema, for example when you have a csv file. + entry.schema.columns.append( + datacatalog_v1.types.ColumnSchema( + column="first_name", + description="First name", + mode="REQUIRED", + type_="STRING", + ) + ) + + entry.schema.columns.append( + datacatalog_v1.types.ColumnSchema( + column="last_name", description="Last name", mode="REQUIRED", type_="STRING" + ) + ) + + # Create the addresses parent column + addresses_column = datacatalog_v1.types.ColumnSchema( + column="addresses", description="Addresses", mode="REPEATED", type_="RECORD" + ) + + # Create sub columns for the addresses parent column + addresses_column.subcolumns.append( + datacatalog_v1.types.ColumnSchema( + column="city", description="City", mode="NULLABLE", type_="STRING" + ) + ) + + addresses_column.subcolumns.append( + datacatalog_v1.types.ColumnSchema( + column="state", description="State", mode="NULLABLE", type_="STRING" + ) + ) + + entry.schema.columns.append(addresses_column) + + entry = datacatalog.create_entry( + parent=entry_group.name, entry_id=fileset_entry_id, entry=entry + ) + print(f"Created fileset entry: {entry.name}") + # [END data_catalog_create_fileset] diff --git a/datacatalog/snippets/create_fileset_test.py b/datacatalog/snippets/create_fileset_test.py new file mode 100644 index 000000000000..d4f928550c21 --- /dev/null +++ b/datacatalog/snippets/create_fileset_test.py @@ -0,0 +1,44 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import create_fileset + + +def test_create_fileset( + capsys, + client, + project_id, + random_entry_group_id, + random_entry_id, + resources_to_delete, +): + location = "us-central1" + override_values = { + "project_id": project_id, + "fileset_entry_group_id": random_entry_group_id, + "fileset_entry_id": random_entry_id, + } + expected_group_name = client.entry_group_path( + project_id, location, random_entry_group_id + ) + expected_entry_name = client.entry_path( + project_id, location, random_entry_group_id, random_entry_id + ) + create_fileset.create_fileset(override_values) + out, err = capsys.readouterr() + assert f"Created entry group: {expected_group_name}" in out + assert f"Created fileset entry: {expected_entry_name}" in out + resources_to_delete["entry_groups"].append(expected_group_name) + resources_to_delete["entries"].append(expected_entry_name) diff --git a/datacatalog/snippets/grant_tag_template_user_role.py b/datacatalog/snippets/grant_tag_template_user_role.py new file mode 100644 index 000000000000..71afec5c361f --- /dev/null +++ b/datacatalog/snippets/grant_tag_template_user_role.py @@ -0,0 +1,69 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def grant_tag_template_user_role(override_values): + """Grants a user the Tag Template User role for a given template.""" + # [START data_catalog_grant_tag_template_user_role] + from google.cloud import datacatalog_v1 + from google.iam.v1 import iam_policy_pb2 as iam_policy + from google.iam.v1 import policy_pb2 + + datacatalog = datacatalog_v1.DataCatalogClient() + + # TODO: Set these values before running the sample. + project_id = "project_id" + tag_template_id = "existing_tag_template_id" + # For a full list of values a member can have, see: + # https://cloud.google.com/iam/docs/reference/rest/v1/Policy?hl=en#binding + member_id = "user:super-cool.test-user@gmail.com" + + # [END data_catalog_grant_tag_template_user_role] + + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + project_id = override_values.get("project_id", project_id) + tag_template_id = override_values.get("tag_template_id", tag_template_id) + member_id = override_values.get("member_id", member_id) + + # [START data_catalog_grant_tag_template_user_role] + # For all regions available, see: + # https://cloud.google.com/data-catalog/docs/concepts/regions + location = "us-central1" + + # Format the Template name. + template_name = datacatalog_v1.DataCatalogClient.tag_template_path( + project_id, location, tag_template_id + ) + + # Retrieve Template's current IAM Policy. + policy = datacatalog.get_iam_policy(resource=template_name) + + # Add Tag Template User role and member to the policy. + binding = policy_pb2.Binding() + binding.role = "roles/datacatalog.tagTemplateUser" + binding.members.append(member_id) + policy.bindings.append(binding) + + set_policy_request = iam_policy.SetIamPolicyRequest( + resource=template_name, policy=policy + ) + + # Update Template's policy. + policy = datacatalog.set_iam_policy(set_policy_request) + + for binding in policy.bindings: + for member in binding.members: + print(f"Member: {member}, Role: {binding.role}") + # [END data_catalog_grant_tag_template_user_role] diff --git a/datacatalog/snippets/grant_tag_template_user_role_test.py b/datacatalog/snippets/grant_tag_template_user_role_test.py new file mode 100644 index 000000000000..005638dd3fca --- /dev/null +++ b/datacatalog/snippets/grant_tag_template_user_role_test.py @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import grant_tag_template_user_role + + +def test_grant_tag_template_user_role( + capsys, project_id, random_existing_tag_template_id, valid_member_id +): + override_values = { + "project_id": project_id, + "tag_template_id": random_existing_tag_template_id, + "member_id": valid_member_id, + } + grant_tag_template_user_role.grant_tag_template_user_role(override_values) + out, err = capsys.readouterr() + assert f"Member: {valid_member_id}, Role: roles/datacatalog.tagTemplateUser" in out diff --git a/datacatalog/snippets/lookup_entry.py b/datacatalog/snippets/lookup_entry.py index 656cb97e6452..110001836b94 100644 --- a/datacatalog/snippets/lookup_entry.py +++ b/datacatalog/snippets/lookup_entry.py @@ -14,139 +14,83 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""This application demonstrates how to perform basic operations on entries -with the Cloud Data Catalog API. -For more information, see the README.md under /datacatalog and the -documentation at https://cloud.google.com/data-catalog/docs. -""" - -import argparse - - -def lookup_bigquery_dataset(project_id, dataset_id): +def lookup_entry(override_values): + """Retrieves Data Catalog entry for the given Google Cloud Platform resource.""" # [START datacatalog_lookup_dataset] - """Retrieves Data Catalog entry for the given BigQuery Dataset.""" - from google.cloud import datacatalog_v1 - - datacatalog = datacatalog_v1.DataCatalogClient() - - resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}'\ - .format(project_id, dataset_id) - - return datacatalog.lookup_entry(request={'linked_resource': resource_name}) - # [END datacatalog_lookup_dataset] - - -def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): - """Retrieves Data Catalog entry for the given BigQuery Dataset by - sql_resource. - """ - from google.cloud import datacatalog_v1 - - datacatalog = datacatalog_v1.DataCatalogClient() - - sql_resource = 'bigquery.dataset.`{}`.`{}`'.format(project_id, dataset_id) - - return datacatalog.lookup_entry(request={'sql_resource': sql_resource}) - - -def lookup_bigquery_table(project_id, dataset_id, table_id): - """Retrieves Data Catalog entry for the given BigQuery Table.""" + # [START data_catalog_lookup_entry] from google.cloud import datacatalog_v1 datacatalog = datacatalog_v1.DataCatalogClient() - resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}' \ - '/tables/{}'\ - .format(project_id, dataset_id, table_id) + bigquery_project_id = "my_bigquery_project" + dataset_id = "my_dataset" + table_id = "my_table" + pubsub_project_id = "my_pubsub_project" + topic_id = "my_topic" - return datacatalog.lookup_entry(request={'linked_resource': resource_name}) + # [END data_catalog_lookup_entry] + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + bigquery_project_id = override_values.get( + "bigquery_project_id", bigquery_project_id + ) + dataset_id = override_values.get("dataset_id", dataset_id) + table_id = override_values.get("table_id", table_id) + pubsub_project_id = override_values.get("pubsub_project_id", pubsub_project_id) + topic_id = override_values.get("topic_id", topic_id) + + # [START data_catalog_lookup_entry] + # BigQuery Dataset via linked_resource + resource_name = f"//bigquery.googleapis.com/projects/{bigquery_project_id}/datasets/{dataset_id}" + + entry = datacatalog.lookup_entry(request={"linked_resource": resource_name}) + print( + f"Retrieved entry {entry.name} for BigQuery Dataset resource {entry.linked_resource}" + ) -def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): - """Retrieves Data Catalog entry for the given BigQuery Table by - sql_resource. - """ - from google.cloud import datacatalog_v1 - - datacatalog = datacatalog_v1.DataCatalogClient() - - sql_resource = 'bigquery.table.`{}`.`{}`.`{}`'.format( - project_id, dataset_id, table_id) - - return datacatalog.lookup_entry(request={'sql_resource': sql_resource}) - - -def lookup_pubsub_topic(project_id, topic_id): - """Retrieves Data Catalog entry for the given Pub/Sub Topic.""" - from google.cloud import datacatalog_v1 - - datacatalog = datacatalog_v1.DataCatalogClient() + # BigQuery Dataset via sql_resource + sql_resource = f"bigquery.dataset.`{bigquery_project_id}`.`{dataset_id}`" - resource_name = '//pubsub.googleapis.com/projects/{}/topics/{}'\ - .format(project_id, topic_id) + entry = datacatalog.lookup_entry(request={"sql_resource": sql_resource}) + print( + f"Retrieved entry {entry.name} for BigQuery Dataset resource {entry.linked_resource}" + ) - return datacatalog.lookup_entry(request={'linked_resource': resource_name}) + # BigQuery Table via linked_resource + resource_name = ( + f"//bigquery.googleapis.com/projects/{bigquery_project_id}/datasets/{dataset_id}" + f"/tables/{table_id}" + ) + entry = datacatalog.lookup_entry(request={"linked_resource": resource_name}) + print(f"Retrieved entry {entry.name} for BigQuery Table {entry.linked_resource}") -def lookup_pubsub_topic_sql_resource(project_id, topic_id): - """Retrieves Data Catalog entry for the given Pub/Sub Topic by - sql_resource. - """ - from google.cloud import datacatalog_v1 + # BigQuery Table via sql_resource + sql_resource = f"bigquery.table.`{bigquery_project_id}`.`{dataset_id}`.`{table_id}`" - datacatalog = datacatalog_v1.DataCatalogClient() + entry = datacatalog.lookup_entry(request={"sql_resource": sql_resource}) + print( + f"Retrieved entry {entry.name} for BigQuery Table resource {entry.linked_resource}" + ) - sql_resource = 'pubsub.topic.`{}`.`{}`'.format(project_id, topic_id) + # Pub/Sub Topic via linked_resource + resource_name = ( + f"//pubsub.googleapis.com/projects/{pubsub_project_id}/topics/{topic_id}" + ) - return datacatalog.lookup_entry(request={'sql_resource': sql_resource}) + entry = datacatalog.lookup_entry(request={"linked_resource": resource_name}) + print( + f"Retrieved entry {entry.name} for Pub/Sub Topic resource {entry.linked_resource}" + ) + # Pub/Sub Topic via sql_resource + sql_resource = f"pubsub.topic.`{pubsub_project_id}`.`{topic_id}`" -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + entry = datacatalog.lookup_entry(request={"sql_resource": sql_resource}) + print( + f"Retrieved entry {entry.name} for Pub/Sub Topic resource {entry.linked_resource}" ) - - parser.add_argument('project_id', help='Your Google Cloud project ID') - - subparsers = parser.add_subparsers(dest='command') - - bigquery_dataset_parser = subparsers.add_parser( - 'bigquery-dataset', help=lookup_bigquery_dataset.__doc__) - bigquery_dataset_parser.add_argument('dataset_id') - bigquery_dataset_parser.add_argument('--sql-resource', action='store_true', - help='Perform lookup by SQL Resource') - - bigquery_table_parser = subparsers.add_parser( - 'bigquery-table', help=lookup_bigquery_table.__doc__) - bigquery_table_parser.add_argument('dataset_id') - bigquery_table_parser.add_argument('table_id') - bigquery_table_parser.add_argument('--sql-resource', action='store_true', - help='Perform lookup by SQL Resource') - - pubsub_topic_parser = subparsers.add_parser( - 'pubsub-topic', help=lookup_pubsub_topic.__doc__) - pubsub_topic_parser.add_argument('topic_id') - pubsub_topic_parser.add_argument('--sql-resource', action='store_true', - help='Perform lookup by SQL Resource') - - args = parser.parse_args() - - entry = None - - if args.command == 'bigquery-dataset': - lookup_method = lookup_bigquery_dataset_sql_resource \ - if args.sql_resource else lookup_bigquery_dataset - entry = lookup_method(args.project_id, args.dataset_id) - elif args.command == 'bigquery-table': - lookup_method = lookup_bigquery_table_sql_resource \ - if args.sql_resource else lookup_bigquery_table - entry = lookup_method(args.project_id, args.dataset_id, args.table_id) - elif args.command == 'pubsub-topic': - lookup_method = lookup_pubsub_topic_sql_resource \ - if args.sql_resource else lookup_pubsub_topic - entry = lookup_method(args.project_id, args.topic_id) - - print(entry.name) + # [END data_catalog_lookup_entry] + # [END datacatalog_lookup_dataset] diff --git a/datacatalog/snippets/lookup_entry_test.py b/datacatalog/snippets/lookup_entry_test.py index 2030cb072197..55245a93f6c9 100644 --- a/datacatalog/snippets/lookup_entry_test.py +++ b/datacatalog/snippets/lookup_entry_test.py @@ -14,40 +14,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -import lookup_entry - -BIGQUERY_PROJECT = 'bigquery-public-data' -BIGQUERY_DATASET = 'new_york_taxi_trips' -BIGQUERY_TABLE = 'taxi_zone_geom' - -PUBSUB_PROJECT = 'pubsub-public-data' -PUBSUB_TOPIC = 'taxirides-realtime' - - -def test_lookup_bigquery_dataset(): - assert lookup_entry.lookup_bigquery_dataset( - BIGQUERY_PROJECT, BIGQUERY_DATASET) - - -def test_lookup_bigquery_dataset_sql_resource(): - assert lookup_entry.lookup_bigquery_dataset_sql_resource( - BIGQUERY_PROJECT, BIGQUERY_DATASET) - - -def test_lookup_bigquery_table(): - assert lookup_entry.lookup_bigquery_table( - BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) - - -def test_lookup_bigquery_table_sql_resource(): - assert lookup_entry.lookup_bigquery_table_sql_resource( - BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) - - -def test_lookup_pubsub_topic(): - assert lookup_entry.lookup_pubsub_topic(PUBSUB_PROJECT, PUBSUB_TOPIC) +import re +import lookup_entry -def test_lookup_pubsub_topic_sql_resource(): - assert lookup_entry.lookup_pubsub_topic_sql_resource( - PUBSUB_PROJECT, PUBSUB_TOPIC) +BIGQUERY_PROJECT = "bigquery-public-data" +BIGQUERY_DATASET = "new_york_taxi_trips" +BIGQUERY_TABLE = "taxi_zone_geom" + +PUBSUB_PROJECT = "pubsub-public-data" +PUBSUB_TOPIC = "taxirides-realtime" + + +def test_lookup_entry(capsys): + override_values = { + "bigquery_project_id": BIGQUERY_PROJECT, + "dataset_id": BIGQUERY_DATASET, + "table_id": BIGQUERY_TABLE, + "pubsub_project_id": PUBSUB_PROJECT, + "topic_id": PUBSUB_TOPIC, + } + dataset_resource = f"//bigquery.googleapis.com/projects/{BIGQUERY_PROJECT}/datasets/{BIGQUERY_DATASET}" + table_resource = f"//bigquery.googleapis.com/projects/{BIGQUERY_PROJECT}/datasets/{BIGQUERY_DATASET}/tables/{BIGQUERY_TABLE}" + topic_resource = ( + f"//pubsub.googleapis.com/projects/{PUBSUB_PROJECT}/topics/{PUBSUB_TOPIC}" + ) + lookup_entry.lookup_entry(override_values) + out, err = capsys.readouterr() + assert re.search( + f"(Retrieved entry .+ for BigQuery Dataset resource {dataset_resource})", out + ) + assert re.search( + f"(Retrieved entry .+ for BigQuery Table resource {table_resource})", out + ) + assert re.search( + f"(Retrieved entry .+ for Pub/Sub Topic resource {topic_resource})", out + ) diff --git a/datacatalog/snippets/noxfile.py b/datacatalog/snippets/noxfile.py index 6a8ccdae22c9..1b0d6c900250 100644 --- a/datacatalog/snippets/noxfile.py +++ b/datacatalog/snippets/noxfile.py @@ -39,17 +39,15 @@ TEST_CONFIG = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - + "ignored_versions": ["2.7"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them - 'enforce_type_hints': False, - + "enforce_type_hints": False, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # If you need to use a specific version of pip, # change pip_version_override to the string representation @@ -57,13 +55,13 @@ "pip_version_override": None, # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } try: # Ensure we can import noxfile_config in the project's directory. - sys.path.append('.') + sys.path.append(".") from noxfile_config import TEST_CONFIG_OVERRIDE except ImportError as e: print("No user noxfile_config found: detail: {}".format(e)) @@ -78,12 +76,12 @@ def get_pytest_env_vars() -> Dict[str, str]: ret = {} # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG['gcloud_project_env'] + env_key = TEST_CONFIG["gcloud_project_env"] # This should error out if not set. - ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] # Apply user supplied envs. - ret.update(TEST_CONFIG['envs']) + ret.update(TEST_CONFIG["envs"]) return ret @@ -92,7 +90,7 @@ def get_pytest_env_vars() -> Dict[str, str]: ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"] # Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) @@ -141,7 +139,7 @@ def _determine_local_import_names(start_dir: str) -> List[str]: @nox.session def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG['enforce_type_hints']: + if not TEST_CONFIG["enforce_type_hints"]: session.install("flake8", "flake8-import-order") else: session.install("flake8", "flake8-import-order", "flake8-annotations") @@ -150,9 +148,11 @@ def lint(session: nox.sessions.Session) -> None: args = FLAKE8_COMMON_ARGS + [ "--application-import-names", ",".join(local_names), - "." + ".", ] session.run("flake8", *args) + + # # Black # @@ -165,6 +165,7 @@ def blacken(session: nox.sessions.Session) -> None: session.run("black", *python_files) + # # Sample Tests # @@ -173,7 +174,9 @@ def blacken(session: nox.sessions.Session) -> None: PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] -def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: if TEST_CONFIG["pip_version_override"]: pip_version = TEST_CONFIG["pip_version_override"] session.install(f"pip=={pip_version}") @@ -203,7 +206,7 @@ def _session_tests(session: nox.sessions.Session, post_install: Callable = None) # on travis where slow and flaky tests are excluded. # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html success_codes=[0, 5], - env=get_pytest_env_vars() + env=get_pytest_env_vars(), ) @@ -213,9 +216,9 @@ def py(session: nox.sessions.Session) -> None: if session.python in TESTED_VERSIONS: _session_tests(session) else: - session.skip("SKIPPED: {} tests are disabled for this sample.".format( - session.python - )) + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) # @@ -224,7 +227,7 @@ def py(session: nox.sessions.Session) -> None: def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ + """Returns the root folder of the project.""" # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): diff --git a/datacatalog/snippets/search_assets.py b/datacatalog/snippets/search_assets.py new file mode 100644 index 000000000000..113acbd2eeb4 --- /dev/null +++ b/datacatalog/snippets/search_assets.py @@ -0,0 +1,48 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def search_assets(override_values): + """Searches Data Catalog entries for a given project.""" + # [START data_catalog_search_assets] + from google.cloud import datacatalog_v1 + + datacatalog = datacatalog_v1.DataCatalogClient() + + # TODO: Set these values before running the sample. + project_id = "project_id" + + # Set custom query. + search_string = "type=dataset" + # [END data_catalog_search_assets] + + # To facilitate testing, we replace values with alternatives + # provided by the testing harness. + project_id = override_values.get("project_id", project_id) + tag_template_id = override_values.get("tag_template_id", search_string) + search_string = f"name:{tag_template_id}" + + # [START data_catalog_search_assets] + scope = datacatalog_v1.types.SearchCatalogRequest.Scope() + scope.include_project_ids.append(project_id) + + # Alternatively, search using organization scopes. + # scope.include_org_ids.append("my_organization_id") + + search_results = datacatalog.search_catalog(scope=scope, query=search_string) + + print("Results in project:") + for result in search_results: + print(result) + # [END data_catalog_search_assets] diff --git a/datacatalog/snippets/search_assets_test.py b/datacatalog/snippets/search_assets_test.py new file mode 100644 index 000000000000..84c266d3397d --- /dev/null +++ b/datacatalog/snippets/search_assets_test.py @@ -0,0 +1,26 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import search_assets + + +def test_search_assets(capsys, project_id, random_existing_tag_template_id): + override_values = { + "project_id": project_id, + "tag_template_id": random_existing_tag_template_id, + } + search_assets.search_assets(override_values) + out, err = capsys.readouterr() + assert "Results in project:" in out + assert random_existing_tag_template_id in out diff --git a/datacatalog/tests/__init__.py b/datacatalog/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/tests/quickstart/__init__.py b/datacatalog/tests/quickstart/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/tests/quickstart/test_create_fileset_entry_quickstart.py b/datacatalog/tests/quickstart/test_create_fileset_entry_quickstart.py deleted file mode 100644 index 769d034fac4a..000000000000 --- a/datacatalog/tests/quickstart/test_create_fileset_entry_quickstart.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from google.cloud import datacatalog_v1beta1 - -from ...quickstart import create_fileset_entry_quickstart - - -def test_create_fileset_entry_quickstart( - capsys, client, project_id, random_entry_group_id, random_entry_id -): - - create_fileset_entry_quickstart.create_fileset_entry_quickstart( - client, project_id, random_entry_group_id, random_entry_id - ) - out, err = capsys.readouterr() - assert ( - "Created entry group" - " projects/{}/locations/{}/entryGroups/{}".format( - project_id, "us-central1", random_entry_group_id - ) - in out - ) - - expected_entry_name = datacatalog_v1beta1.DataCatalogClient.entry_path( - project_id, "us-central1", random_entry_group_id, random_entry_id - ) - - assert "Created entry {}".format(expected_entry_name) in out diff --git a/datacatalog/v1beta1/__init__.py b/datacatalog/v1beta1/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/tests/conftest.py b/datacatalog/v1beta1/conftest.py similarity index 72% rename from datacatalog/tests/conftest.py rename to datacatalog/v1beta1/conftest.py index 6ee1fcb621ea..d641d1359132 100644 --- a/datacatalog/tests/conftest.py +++ b/datacatalog/v1beta1/conftest.py @@ -16,11 +16,11 @@ import datetime import uuid -import pytest - import google.auth from google.cloud import datacatalog_v1beta1 +import pytest + @pytest.fixture(scope="session") def client(credentials): @@ -52,7 +52,7 @@ def random_entry_id(client, project_id, random_entry_group_id): entry_name = datacatalog_v1beta1.DataCatalogClient.entry_path( project_id, "us-central1", random_entry_group_id, random_entry_id ) - client.delete_entry(request = {'name': entry_name}) + client.delete_entry(request={"name": entry_name}) @pytest.fixture @@ -65,7 +65,7 @@ def random_entry_group_id(client, project_id): entry_group_name = datacatalog_v1beta1.DataCatalogClient.entry_group_path( project_id, "us-central1", random_entry_group_id ) - client.delete_entry_group(request = {'name': entry_group_name}) + client.delete_entry_group(request={"name": entry_group_name}) @pytest.fixture @@ -76,7 +76,21 @@ def random_entry_name(client, entry_group_name): ) random_entry_name = "{}/entries/{}".format(entry_group_name, random_entry_id) yield random_entry_name - client.delete_entry(request = {'name': random_entry_name}) + client.delete_entry(request={"name": random_entry_name}) + + +@pytest.fixture +def entry(client, entry_group_name): + now = datetime.datetime.now() + random_entry_id = "example_entry_{}_{}".format( + now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] + ) + entry = datacatalog_v1beta1.CreateEntryRequest + entry = client.create_entry( + request={"parent": entry_group_name, "entry_id": random_entry_id, "entry": {"type_": "DATA_STREAM", "name": "samples_test_entry"}} + ) + yield entry.name + client.delete_entry(request={"name": entry.name}) @pytest.fixture @@ -86,6 +100,11 @@ def entry_group_name(client, project_id): now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8] ) entry_group = client.create_entry_group( - request = {'parent': datacatalog_v1beta1.DataCatalogClient.location_path(project_id, "us-central1"), 'entry_group_id': entry_group_id, 'entry_group': {}}) + request={ + "parent": f"projects/{project_id}/locations/us-central1", + "entry_group_id": entry_group_id, + "entry_group": datacatalog_v1beta1.EntryGroup(), + } + ) yield entry_group.name - client.delete_entry_group(request = {'name': entry_group.name}) + client.delete_entry_group(request={"name": entry_group.name}) diff --git a/datacatalog/v1beta1/create_entry_group.py b/datacatalog/v1beta1/create_entry_group.py index d2056ec63d2c..452f1ded681d 100644 --- a/datacatalog/v1beta1/create_entry_group.py +++ b/datacatalog/v1beta1/create_entry_group.py @@ -13,13 +13,11 @@ # limitations under the License. -def create_entry_group(client, project_id, entry_group_id): - - # [START datacatalog_create_entry_group_tag] +def create_entry_group(project_id, entry_group_id): + # [START data_catalog_create_entry_group_v1beta1] from google.cloud import datacatalog_v1beta1 - # TODO(developer): Construct a Data Catalog client object. - # client = datacatalog_v1beta1.DataCatalogClient() + client = datacatalog_v1beta1.DataCatalogClient() # TODO(developer): Set entry_group_id to the ID of the # entry group to create. @@ -35,9 +33,7 @@ def create_entry_group(client, project_id, entry_group_id): # entry_group_id = "your_entry_group_id" # Construct a full location path to be the parent of the entry group. - parent = datacatalog_v1beta1.DataCatalogClient.location_path( - project_id, location_id - ) + parent = f"projects/{project_id}/locations/{location_id}" # Construct a full EntryGroup object to send to the API. entry_group = datacatalog_v1beta1.EntryGroup() @@ -48,6 +44,11 @@ def create_entry_group(client, project_id, entry_group_id): # Raises google.api_core.exceptions.AlreadyExists if the Entry Group # already exists within the project. entry_group = client.create_entry_group( - request = {'parent': parent, 'entry_group_id': entry_group_id, 'entry_group': entry_group}) # Make an API request. + request={ + "parent": parent, + "entry_group_id": entry_group_id, + "entry_group": entry_group, + } + ) # Make an API request. print("Created entry group {}".format(entry_group.name)) - # [END datacatalog_create_entry_group_tag] + # [END data_catalog_create_entry_group_v1beta1] diff --git a/datacatalog/v1beta1/create_fileset_entry.py b/datacatalog/v1beta1/create_fileset_entry.py index f96255b2bcd8..f798bfb6810b 100644 --- a/datacatalog/v1beta1/create_fileset_entry.py +++ b/datacatalog/v1beta1/create_fileset_entry.py @@ -14,8 +14,7 @@ def create_fileset_entry(client, entry_group_name, entry_id): - - # [START datacatalog_create_fileset_tag] + # [START data_catalog_create_fileset_v1beta1] from google.cloud import datacatalog_v1beta1 # TODO(developer): Construct a Data Catalog client object. @@ -33,7 +32,7 @@ def create_fileset_entry(client, entry_group_name, entry_id): entry.display_name = "My Fileset" entry.description = "This Fileset consists of ..." entry.gcs_fileset_spec.file_patterns.append("gs://my_bucket/*") - entry.type = datacatalog_v1beta1.enums.EntryType.FILESET + entry.type_ = datacatalog_v1beta1.EntryType.FILESET # Create the Schema, for example when you have a csv file. columns = [] @@ -42,13 +41,13 @@ def create_fileset_entry(client, entry_group_name, entry_id): column="first_name", description="First name", mode="REQUIRED", - type="STRING", + type_="STRING", ) ) columns.append( datacatalog_v1beta1.types.ColumnSchema( - column="last_name", description="Last name", mode="REQUIRED", type="STRING" + column="last_name", description="Last name", mode="REQUIRED", type_="STRING" ) ) @@ -56,13 +55,13 @@ def create_fileset_entry(client, entry_group_name, entry_id): subcolumns = [] subcolumns.append( datacatalog_v1beta1.types.ColumnSchema( - column="city", description="City", mode="NULLABLE", type="STRING" + column="city", description="City", mode="NULLABLE", type_="STRING" ) ) subcolumns.append( datacatalog_v1beta1.types.ColumnSchema( - column="state", description="State", mode="NULLABLE", type="STRING" + column="state", description="State", mode="NULLABLE", type_="STRING" ) ) @@ -72,7 +71,7 @@ def create_fileset_entry(client, entry_group_name, entry_id): description="Addresses", mode="REPEATED", subcolumns=subcolumns, - type="RECORD", + type_="RECORD", ) ) @@ -81,6 +80,8 @@ def create_fileset_entry(client, entry_group_name, entry_id): # Send the entry to the API for creation. # Raises google.api_core.exceptions.AlreadyExists if the Entry already # exists within the project. - entry = client.create_entry(request = {'parent': entry_group_name, 'entry_id': entry_id, 'entry': entry}) + entry = client.create_entry( + request={"parent": entry_group_name, "entry_id": entry_id, "entry": entry} + ) print("Created entry {}".format(entry.name)) - # [END datacatalog_create_fileset_tag] + # [END data_catalog_create_fileset_v1beta1] diff --git a/datacatalog/v1beta1/datacatalog_get_entry.py b/datacatalog/v1beta1/get_entry.py similarity index 59% rename from datacatalog/v1beta1/datacatalog_get_entry.py rename to datacatalog/v1beta1/get_entry.py index 05bc0dd52aa3..30b13d0a5d7e 100644 --- a/datacatalog/v1beta1/datacatalog_get_entry.py +++ b/datacatalog/v1beta1/get_entry.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# DO NOT EDIT! This is a generated sample ("Request", "datacatalog_get_entry") - # To install the latest published package dependency, execute the following: # pip install google-cloud-datacatalog @@ -24,18 +22,20 @@ # description: Get Entry # usage: python3 samples/v1beta1/datacatalog_get_entry.py [--project_id "[Google Cloud Project ID]"] [--location_id "[Google Cloud Location ID]"] [--entry_group_id "[Entry Group ID]"] [--entry_id "[Entry ID]"] -# [START datacatalog_get_entry] -from google.cloud import datacatalog_v1beta1 -def sample_get_entry(project_id, location_id, entry_group_id, entry_id): +def sample_get_entry( + project_id: str, location_id: str, entry_group_id: str, entry_id: str +): + # [START data_catalog_get_entry_v1beta1] + from google.cloud import datacatalog_v1beta1 """ Get Entry Args: - project_id Your Google Cloud project ID - location_id Google Cloud region, e.g. us-central1 - entry_group_id ID of the Entry Group, e.g. @bigquery, @pubsub, my_entry_group - entry_id ID of the Entry + project_id (str): Your Google Cloud project ID + location_id (str): Google Cloud region, e.g. us-central1 + entry_group_id (str): ID of the Entry Group, e.g. @bigquery, @pubsub, my_entry_group + entry_id (str): ID of the Entry """ client = datacatalog_v1beta1.DataCatalogClient() @@ -46,24 +46,22 @@ def sample_get_entry(project_id, location_id, entry_group_id, entry_id): # entry_id = '[Entry ID]' name = client.entry_path(project_id, location_id, entry_group_id, entry_id) - response = client.get_entry(request = {'name': name}) - entry = response - print(u"Entry name: {}".format(entry.name)) - print(u"Entry type: {}".format(datacatalog_v1beta1.EntryType(entry.type).name)) - print(u"Linked resource: {}".format(entry.linked_resource)) - - -# [END datacatalog_get_entry] + entry = client.get_entry(request={"name": name}) + print(f"Entry name: {entry.name}") + print(f"Entry type: {datacatalog_v1beta1.EntryType(entry.type_).name}") + print(f"Linked resource: {entry.linked_resource}") + # [END data_catalog_get_entry_v1beta1] + return entry def main(): import argparse parser = argparse.ArgumentParser() - parser.add_argument("--project_id", type=str, default="[Google Cloud Project ID]") - parser.add_argument("--location_id", type=str, default="[Google Cloud Location ID]") - parser.add_argument("--entry_group_id", type=str, default="[Entry Group ID]") - parser.add_argument("--entry_id", type=str, default="[Entry ID]") + parser.add_argument("--project_id", type_=str, default="[Google Cloud Project ID]") + parser.add_argument("--location_id", type_=str, default="[Google Cloud Location ID]") + parser.add_argument("--entry_group_id", type_=str, default="[Entry Group ID]") + parser.add_argument("--entry_id", type_=str, default="[Entry ID]") args = parser.parse_args() sample_get_entry( diff --git a/datacatalog/v1beta1/datacatalog_lookup_entry.py b/datacatalog/v1beta1/lookup_entry.py similarity index 66% rename from datacatalog/v1beta1/datacatalog_lookup_entry.py rename to datacatalog/v1beta1/lookup_entry.py index 176d080db766..a167789126d0 100644 --- a/datacatalog/v1beta1/datacatalog_lookup_entry.py +++ b/datacatalog/v1beta1/lookup_entry.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# DO NOT EDIT! This is a generated sample ("Request", "datacatalog_lookup_entry") - # To install the latest published package dependency, execute the following: # pip install google-cloud-datacatalog @@ -24,16 +22,15 @@ # description: Lookup Entry # usage: python3 samples/v1beta1/datacatalog_lookup_entry.py [--resource_name "[Full Resource Name]"] -# [START datacatalog_lookup_entry] -from google.cloud import datacatalog_v1beta1 - -def sample_lookup_entry(resource_name): +def sample_lookup_entry(resource_name: str): + # [START data_catalog_lookup_entry_v1beta1] + from google.cloud import datacatalog_v1beta1 """ Lookup Entry Args: - resource_name The full name of the Google Cloud Platform resource the Data + resource_name (str): The full name of the Google Cloud Platform resource the Data Catalog entry represents. See: https://cloud.google.com/apis/design/resource_names#full_resource_name Examples: @@ -42,23 +39,19 @@ def sample_lookup_entry(resource_name): """ client = datacatalog_v1beta1.DataCatalogClient() - - # resource_name = '[Full Resource Name]' - response = client.lookup_entry(request = {'linked_resource': resource_name}) - entry = response - print(u"Entry name: {}".format(entry.name)) - print(u"Entry type: {}".format(datacatalog_v1beta1.EntryType(entry.type).name)) - print(u"Linked resource: {}".format(entry.linked_resource)) - - -# [END datacatalog_lookup_entry] + entry = client.lookup_entry(request={"linked_resource": resource_name}) + print(f"Entry name: {entry.name}") + print(f"Entry type: {datacatalog_v1beta1.EntryType(entry.type_).name}") + print(f"Linked resource: {entry.linked_resource}") + # [END data_catalog_lookup_entry_v1beta1] + return entry def main(): import argparse parser = argparse.ArgumentParser() - parser.add_argument("--resource_name", type=str, default="[Full Resource Name]") + parser.add_argument("--resource_name", type_=str, default="[Full Resource Name]") args = parser.parse_args() sample_lookup_entry(args.resource_name) diff --git a/datacatalog/v1beta1/datacatalog_lookup_entry_sql_resource.py b/datacatalog/v1beta1/lookup_entry_sql_resource.py similarity index 70% rename from datacatalog/v1beta1/datacatalog_lookup_entry_sql_resource.py rename to datacatalog/v1beta1/lookup_entry_sql_resource.py index f46af3698080..dd6de7867ae0 100644 --- a/datacatalog/v1beta1/datacatalog_lookup_entry_sql_resource.py +++ b/datacatalog/v1beta1/lookup_entry_sql_resource.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# DO NOT EDIT! This is a generated sample ("Request", "datacatalog_lookup_entry_sql_resource") - # To install the latest published package dependency, execute the following: # pip install google-cloud-datacatalog @@ -24,16 +22,15 @@ # description: Lookup Entry using SQL resource # usage: python3 samples/v1beta1/datacatalog_lookup_entry_sql_resource.py [--sql_name "[SQL Resource Name]"] -# [START datacatalog_lookup_entry_sql_resource] -from google.cloud import datacatalog_v1beta1 - -def sample_lookup_entry(sql_name): +def sample_lookup_entry(sql_name: str): + # [START data_catalog_lookup_entry_sql_resource_v1beta1] + from google.cloud import datacatalog_v1beta1 """ Lookup Entry using SQL resource Args: - sql_name The SQL name of the Google Cloud Platform resource the Data Catalog + sql_name (str): The SQL name of the Google Cloud Platform resource the Data Catalog entry represents. Examples: bigquery.table.`bigquery-public-data`.new_york_taxi_trips.taxi_zone_geom @@ -43,14 +40,12 @@ def sample_lookup_entry(sql_name): client = datacatalog_v1beta1.DataCatalogClient() # sql_name = '[SQL Resource Name]' - response = client.lookup_entry(request = {'sql_resource': sql_name}) - entry = response - print(u"Entry name: {}".format(entry.name)) - print(u"Entry type: {}".format(datacatalog_v1beta1.EntryType(entry.type).name)) - print(u"Linked resource: {}".format(entry.linked_resource)) - - -# [END datacatalog_lookup_entry_sql_resource] + entry = client.lookup_entry(request={"sql_resource": sql_name}) + print(f"Entry name: {entry.name}") + print(f"Entry type: {datacatalog_v1beta1.EntryType(entry.type_).name}") + print(f"Linked resource: {entry.linked_resource}") + # [END data_catalog_lookup_entry_sql_resource_v1beta1] + return entry def main(): diff --git a/datacatalog/v1beta1/noxfile.py b/datacatalog/v1beta1/noxfile.py new file mode 100644 index 000000000000..5660f08be441 --- /dev/null +++ b/datacatalog/v1beta1/noxfile.py @@ -0,0 +1,222 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/datacatalog/v1beta1/requirements-test.txt b/datacatalog/v1beta1/requirements-test.txt new file mode 100644 index 000000000000..95ea1e6a02b0 --- /dev/null +++ b/datacatalog/v1beta1/requirements-test.txt @@ -0,0 +1 @@ +pytest==6.2.4 diff --git a/datacatalog/v1beta1/requirements.txt b/datacatalog/v1beta1/requirements.txt new file mode 100644 index 000000000000..5f898b7ae101 --- /dev/null +++ b/datacatalog/v1beta1/requirements.txt @@ -0,0 +1 @@ +google-cloud-datacatalog==3.4.0 diff --git a/datacatalog/v1beta1/datacatalog_search.py b/datacatalog/v1beta1/search.py similarity index 68% rename from datacatalog/v1beta1/datacatalog_search.py rename to datacatalog/v1beta1/search.py index ad10276698a4..f4893083589c 100644 --- a/datacatalog/v1beta1/datacatalog_search.py +++ b/datacatalog/v1beta1/search.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# DO NOT EDIT! This is a generated sample ("RequestPagedAll", "datacatalog_search") - # To install the latest published package dependency, execute the following: # pip install google-cloud-datacatalog @@ -24,20 +22,20 @@ # description: Search Catalog # usage: python3 samples/v1beta1/datacatalog_search.py [--include_project_id "[Google Cloud Project ID]"] [--include_gcp_public_datasets false] [--query "[String in search query syntax]"] -# [START datacatalog_search] -from google.cloud import datacatalog_v1beta1 -from google.cloud.datacatalog_v1beta1 import enums - -def sample_search_catalog(include_project_id, include_gcp_public_datasets, query): +def sample_search_catalog( + include_project_id: str, include_gcp_public_datasets: bool, query: str +): + # [START data_catalog_search_v1beta1] + from google.cloud import datacatalog_v1beta1 """ Search Catalog Args: - include_project_id Your Google Cloud project ID. - include_gcp_public_datasets If true, include Google Cloud Platform (GCP) public + include_project_id (str): Your Google Cloud project ID. + include_gcp_public_datasets (bool): If true, include Google Cloud Platform (GCP) public datasets in the search results. - query Your query string. + query (str): Your query string. See: https://cloud.google.com/data-catalog/docs/how-to/search-reference Example: system=bigquery type=dataset """ @@ -54,20 +52,16 @@ def sample_search_catalog(include_project_id, include_gcp_public_datasets, query } # Iterate over all results - for response_item in client.search_catalog(request = {'scope': scope, 'query': query}): - print( - u"Result type: {}".format( - enums.SearchResultType(response_item.search_result_type).name - ) - ) - print(u"Result subtype: {}".format(response_item.search_result_subtype)) + results = client.search_catalog(request={"scope": scope, "query": query}) + for response_item in results: print( - u"Relative resource name: {}".format(response_item.relative_resource_name) + f"Result type: {datacatalog_v1beta1.SearchResultType(response_item.search_result_type).name}" ) - print(u"Linked resource: {}\n".format(response_item.linked_resource)) - - -# [END datacatalog_search] + print(f"Result subtype: {response_item.search_result_subtype}") + print(f"Relative resource name: {response_item.relative_resource_name}") + print(f"Linked resource: {response_item.linked_resource}\n") + # [END data_catalog_search_v1beta1] + return results def main(): diff --git a/datacatalog/tests/test_create_entry_group.py b/datacatalog/v1beta1/test_create_entry_group.py similarity index 83% rename from datacatalog/tests/test_create_entry_group.py rename to datacatalog/v1beta1/test_create_entry_group.py index 443c97f92921..f7fe80cc025c 100644 --- a/datacatalog/tests/test_create_entry_group.py +++ b/datacatalog/v1beta1/test_create_entry_group.py @@ -13,12 +13,12 @@ # limitations under the License. -from ..v1beta1 import create_entry_group +import create_entry_group def test_create_entry_group(capsys, client, project_id, random_entry_group_id): - create_entry_group.create_entry_group(request = {'parent': client, 'entry_group_id': project_id, 'entry_group': random_entry_group_id}) + create_entry_group.create_entry_group(project_id, random_entry_group_id) out, err = capsys.readouterr() assert ( "Created entry group" diff --git a/datacatalog/tests/test_create_fileset_entry.py b/datacatalog/v1beta1/test_create_fileset_entry.py similarity index 96% rename from datacatalog/tests/test_create_fileset_entry.py rename to datacatalog/v1beta1/test_create_fileset_entry.py index 8d0bc28fd07f..b9af5d8c3706 100644 --- a/datacatalog/tests/test_create_fileset_entry.py +++ b/datacatalog/v1beta1/test_create_fileset_entry.py @@ -15,7 +15,7 @@ import re -from ..v1beta1 import create_fileset_entry +import create_fileset_entry def test_create_fileset_entry(capsys, client, random_entry_name): diff --git a/datacatalog/v1beta1/test_get_entry.py b/datacatalog/v1beta1/test_get_entry.py new file mode 100644 index 000000000000..70d703a52a86 --- /dev/null +++ b/datacatalog/v1beta1/test_get_entry.py @@ -0,0 +1,25 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import get_entry + + +def test_get_entry(client, entry): + # break entry name into parts + name = client.parse_entry_path(entry) + retrieved_entry = get_entry.sample_get_entry( + name["project"], name["location"], name["entry_group"], name["entry"] + ) + assert retrieved_entry.name == entry diff --git a/datacatalog/v1beta1/test_lookup_entry.py b/datacatalog/v1beta1/test_lookup_entry.py new file mode 100644 index 000000000000..5091cd2b0255 --- /dev/null +++ b/datacatalog/v1beta1/test_lookup_entry.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import lookup_entry + +BIGQUERY_PROJECT = "bigquery-public-data" +BIGQUERY_DATASET = "new_york_taxi_trips" + + +def test_lookup_entry(client, entry, project_id): + bigquery_dataset = f"projects/{BIGQUERY_PROJECT}/datasets/{BIGQUERY_DATASET}" + resource_name = f"//bigquery.googleapis.com/{bigquery_dataset}" + + found_entry = lookup_entry.sample_lookup_entry(resource_name) + assert found_entry.linked_resource == resource_name diff --git a/datacatalog/v1beta1/test_lookup_entry_sql_resource.py b/datacatalog/v1beta1/test_lookup_entry_sql_resource.py new file mode 100644 index 000000000000..daf45523a2a1 --- /dev/null +++ b/datacatalog/v1beta1/test_lookup_entry_sql_resource.py @@ -0,0 +1,26 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import lookup_entry_sql_resource + +BIGQUERY_PROJECT = "bigquery-public-data" +BIGQUERY_DATASET = "new_york_taxi_trips" + + +def test_lookup_entry(): + sql_name = f"bigquery.dataset.`{BIGQUERY_PROJECT}`.`{BIGQUERY_DATASET}`" + resource_name = f"//bigquery.googleapis.com/projects/{BIGQUERY_PROJECT}/datasets/{BIGQUERY_DATASET}" + entry = lookup_entry_sql_resource.sample_lookup_entry(sql_name) + assert entry.linked_resource == resource_name diff --git a/datacatalog/v1beta1/test_search.py b/datacatalog/v1beta1/test_search.py new file mode 100644 index 000000000000..48fe6cf46384 --- /dev/null +++ b/datacatalog/v1beta1/test_search.py @@ -0,0 +1,21 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import search + + +def test_search_catalog(client, project_id, entry_group_name): + results = search.sample_search_catalog(project_id, False, f"name:{entry_group_name}") + assert results is not None