Skip to content

Commit

Permalink
samples: Add export to BigQuery (#70)
Browse files Browse the repository at this point in the history
  • Loading branch information
lvvvvvf authored and dandhlee committed Nov 29, 2022
1 parent 054a302 commit bd0020c
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 2 deletions.
26 changes: 26 additions & 0 deletions asset/snippets/snippets/quickstart_exportassets.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,32 @@ def export_assets(project_id, dump_file_path):
# [END asset_quickstart_export_assets]


def export_assets_bigquery(project_id, dataset, table):
# [START asset_quickstart_export_assets_bigquery]
from google.cloud import asset_v1

# TODO project_id = 'Your Google Cloud Project ID'
# TODO dataset = 'Your BigQuery dataset path'
# TODO table = 'Your BigQuery table name'

client = asset_v1.AssetServiceClient()
parent = "projects/{}".format(project_id)
content_type = asset_v1.ContentType.RESOURCE
output_config = asset_v1.OutputConfig()
output_config.bigquery_destination.dataset = dataset
output_config.bigquery_destination.table = table
output_config.bigquery_destination.force = True
response = client.export_assets(
request={
"parent": parent,
"content_type": content_type,
"output_config": output_config
}
)
print(response.result())
# [END asset_quickstart_export_assets_bigquery]


if __name__ == "__main__":

parser = argparse.ArgumentParser(
Expand Down
29 changes: 27 additions & 2 deletions asset/snippets/snippets/quickstart_exportassets_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,27 @@
import os
import uuid

from google.cloud import bigquery
from google.cloud import storage
import pytest

import quickstart_exportassets

PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
BUCKET = "assets-{}".format(uuid.uuid4().hex)
DATASET = "assets_{}".format(int(uuid.uuid4()))


@pytest.fixture(scope="module")
def storage_client():
yield storage.Client()


@pytest.fixture(scope="module")
def bigquery_client():
yield bigquery.Client()


@pytest.fixture(scope="module")
def asset_bucket(storage_client):
bucket = storage_client.create_bucket(BUCKET)
Expand All @@ -44,9 +51,27 @@ def asset_bucket(storage_client):
raise e


def test_export_assets(asset_bucket, capsys):
@pytest.fixture(scope='module')
def dataset(bigquery_client):
dataset_id = "{}.{}".format(PROJECT, DATASET)
dataset = bigquery.Dataset(dataset_id)
dataset.location = "US"
dataset = bigquery_client.create_dataset(dataset)

yield DATASET

bigquery_client.delete_dataset(
dataset_id, delete_contents=True, not_found_ok=False)


def test_export_assets(asset_bucket, dataset, capsys):
dump_file_path = "gs://{}/assets-dump.txt".format(asset_bucket)
quickstart_exportassets.export_assets(PROJECT, dump_file_path)
out, _ = capsys.readouterr()

assert dump_file_path in out

dataset_id = "projects/{}/datasets/{}".format(PROJECT, dataset)
quickstart_exportassets.export_assets_bigquery(
PROJECT, dataset_id, "assettable")
out, _ = capsys.readouterr()
assert dataset_id in out

0 comments on commit bd0020c

Please sign in to comment.