Task/add create fileset entry

.kokoro/datacatalog/cloud-client/lookup-entry.cfg → .kokoro/datacatalog/cloud-client/datacatalog.cfg

@@ -12,14 +12,20 @@ env_vars: {
     value: "github/nodejs-docs-samples/.kokoro/build.sh"
 }
 
-# Specify the project that contains a valid dataset.
+# Specify the project that datacatalog will run on.
 env_vars: {
 	key: "GCLOUD_PROJECT"
+	value: "nodejs-docs-samples-tests"
+}
+
+# Specify the project that contains a valid dataset.
+env_vars: {
+	key: "GCLOUD_PUBLIC_PROJECT"
 	value: "bigquery-public-data"
 }
 
 # Specify the dataset related to the project.
 env_vars: {
-	key: "GCLOUD_DATASET_ID"
+	key: "GCLOUD_PUBLIC_DATASET_ID"
 	value: "new_york_taxi_trips"
 }

datacatalog/cloud-client/createEntryGroup.js

@@ -0,0 +1,65 @@
+/* eslint-disable no-warning-comments */
+
+/**
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/**
+ * This application demonstrates how to create a Entry Group with the
+ * Cloud Data Catalog API.
+
+ * For more information, see the README.md under /datacatalog and the
+ * documentation at https://cloud.google.com/data-catalog/docs.
+ */
+const main = async (projectId = process.env.GCLOUD_PROJECT, entryGroupId) => {
+  // [START datacatalog_create_entry_group_tag]
+  // -------------------------------
+  // Import required modules.
+  // -------------------------------
+  const {DataCatalogClient} = require('@google-cloud/datacatalog').v1beta1;
+  const datacatalog = new DataCatalogClient();
+
+  // Currently, Data Catalog stores metadata in the
+  // us-central1 region.
+  const location = 'us-central1';
+
+  // TODO(developer): Uncomment the following lines before running the sample.
+  // const projectId = 'my-project'  
+  // const entryGroupId = 'my-entry-group'
+
+  // Create an Entry Group.
+  // Construct the EntryGroup for the EntryGroup request.
+  const entryGroup = {
+    displayName: 'My Fileset Entry Group',
+    description: 'This Entry Group consists of ....',
+  };
+
+  // Construct the EntryGroup request to be sent by the client.
+  const entryGroupRequest = {
+    parent: datacatalog.locationPath(projectId, location),
+    entryGroupId: entryGroupId,
+    entryGroup: entryGroup,
+  };
+
+  // Use the client to send the API request.
+  const [response] = await datacatalog.createEntryGroup(entryGroupRequest);
+
+  console.log(response);
+  // [END datacatalog_create_entry_group_tag]
+};
+
+// node createEntryGroup.js <projectId> <entryGroupId>
+main(...process.argv.slice(2));

datacatalog/cloud-client/createFilesetEntry.js

@@ -0,0 +1,110 @@
+/* eslint-disable no-warning-comments */
+
+/**
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/**
+ * This application demonstrates how to create a fileset Entry with the
+ * Cloud Data Catalog API.
+
+ * For more information, see the README.md under /datacatalog and the
+ * documentation at https://cloud.google.com/data-catalog/docs.
+ */
+const main = async (
+  projectId = process.env.GCLOUD_PROJECT,
+  entryGroupId,
+  entryId
+) => {
+  // [START datacatalog_create_fileset_tag]
+  // -------------------------------
+  // Import required modules.
+  // -------------------------------
+  const {DataCatalogClient} = require('@google-cloud/datacatalog').v1beta1;
+  const datacatalog = new DataCatalogClient();
+
+  // Currently, Data Catalog stores metadata in the
+  // us-central1 region.
+  const location = 'us-central1';
+
+  // TODO(developer): Uncomment the following lines before running the sample.
+  // const projectId = 'my-project'    
+  // const entryGroupId = 'my-entry-group'
+  // const entryId = 'my-entry'
+
+  // Create a Fileset Entry.
+  // Construct the Entry for the Entry request.
+  const FILESET_TYPE = 4;
+
+  const entry = {
+    displayName: 'My Fileset',
+    description: 'This fileset consists of ....',
+    gcsFilesetSpec: {filePatterns: ['gs://my_bucket/*']},
+    schema: {
+      columns: [
+        {
+          column: 'city',
+          description: 'City',
+          mode: 'NULLABLE',
+          type: 'STRING',
+        },
+        {
+          column: 'state',
+          description: 'State',
+          mode: 'NULLABLE',
+          type: 'STRING',
+        },
+        {
+          column: 'addresses',
+          description: 'Addresses',
+          mode: 'REPEATED',
+          subcolumns: [
+            {
+              column: 'city',
+              description: 'City',
+              mode: 'NULLABLE',
+              type: 'STRING',
+            },
+            {
+              column: 'state',
+              description: 'State',
+              mode: 'NULLABLE',
+              type: 'STRING',
+            },
+          ],
+          type: 'RECORD',
+        },
+      ],
+    },
+    type: FILESET_TYPE,
+  };
+
+  // Construct the Entry request to be sent by the client.
+  const request = {
+    parent: datacatalog.entryGroupPath(projectId, location, entryGroupId),
+    entryId: entryId,
+    entry: entry,
+  };
+
+  // Use the client to send the API request.
+  const [response] = await datacatalog.createEntry(request);
+
+  console.log(response);
+  // [END datacatalog_create_fileset_tag]
+};
+
+// node createFilesetEntry.js <projectId> <entryGroupId> <entryId>
+main(...process.argv.slice(2));

datacatalog/cloud-client/package.json

@@ -14,10 +14,11 @@
   },
   "devDependencies": {
     "@google-cloud/nodejs-repo-tools": "^3.3.0",
-    "mocha": "^6.0.0"
+    "mocha": "^6.0.0",
+    "uuid": "^3.1.0"
   },
   "dependencies": {
-    "@google-cloud/datacatalog": "^1.0.1"
+    "@google-cloud/datacatalog": "^1.3.0"
   },
   "cloud-repo-tools": {
     "requiresKeyFile": true,

datacatalog/cloud-client/system-test/createEntryGroup.test.js

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const path = require('path');
+const assert = require('assert');
+const tools = require('@google-cloud/nodejs-repo-tools');
+const uuid = require('uuid');
+const cwd = path.join(__dirname, '..');
+
+const projectId = process.env.GCLOUD_PROJECT;
+// Use unique id to avoid conflicts between concurrent test runs
+const entryGroupId = `fileset_entry_group_${uuid.v4().substr(0, 8)}`;
+const location = 'us-central1';
+
+const {DataCatalogClient} = require('@google-cloud/datacatalog').v1beta1;
+const datacatalog = new DataCatalogClient();
+
+before(tools.checkCredentials);
+
+describe('createEntryGroup', () => {
+  it('should create a entry group', async () => {
+    const output = await tools.runAsync(
+      `node createEntryGroup.js ${projectId} ${entryGroupId}`,
+      cwd
+    );
+    const expectedName = `projects/${projectId}/locations/${location}/entryGroups/${entryGroupId}`;
+    assert.ok(output.includes(expectedName));
+  });
+
+  after(async () => {
+    const entryGroupPath = datacatalog.entryGroupPath(
+      projectId,
+      location,
+      entryGroupId
+    );
+    await datacatalog.deleteEntryGroup({name: entryGroupPath});
+  });
+});

datacatalog/cloud-client/system-test/createFilesetEntry.test.js

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const path = require('path');
+const assert = require('assert');
+const tools = require('@google-cloud/nodejs-repo-tools');
+const uuid = require('uuid');
+const cwd = path.join(__dirname, '..');
+
+const projectId = process.env.GCLOUD_PROJECT;
+// Use unique id to avoid conflicts between concurrent test runs
+const entryGroupId = `fileset_entry_group_${uuid.v4().substr(0, 8)}`;
+const entryId = `fileset_entry_id_${uuid.v4().substr(0, 8)}`;
+const location = 'us-central1';
+
+const {DataCatalogClient} = require('@google-cloud/datacatalog').v1beta1;
+const datacatalog = new DataCatalogClient();
+
+before(tools.checkCredentials);
+
+describe('createFilesetEntry', () => {
+  before(async () => {
+    // Must create entryGroup before creating entry
+    await tools.runAsync(
+      `node createEntryGroup.js ${projectId} ${entryGroupId}`,
+      cwd
+    );
+  });
+
+  it('should create a fileset entry', async () => {
+    const output = await tools.runAsync(
+      `node createFilesetEntry.js ${projectId} ${entryGroupId} ${entryId}`,
+      cwd
+    );
+    const expectedLinkedResource = `//datacatalog.googleapis.com/projects/${projectId}/locations/${location}/entryGroups/${entryGroupId}/entries/${entryId}`;
+    assert.ok(output.includes(expectedLinkedResource));
+  });
+
+  after(async () => {
+    const entryPath = datacatalog.entryPath(
+      projectId,
+      location,
+      entryGroupId,
+      entryId
+    );
+    await datacatalog.deleteEntry({name: entryPath});
+    const entryGroupPath = datacatalog.entryGroupPath(
+      projectId,
+      location,
+      entryGroupId
+    );
+    await datacatalog.deleteEntryGroup({name: entryGroupPath});
+  });
+});

datacatalog/cloud-client/system-test/datacatalog.test.js → datacatalog/cloud-client/system-test/lookupEntry.test.js

@@ -18,18 +18,19 @@
 const path = require('path');
 const assert = require('assert');
 const tools = require('@google-cloud/nodejs-repo-tools');
-
 const cwd = path.join(__dirname, '..');
-const projectId = process.env.GCLOUD_PROJECT;
-const datasetId = process.env.GCLOUD_DATASET_ID;
 
 before(tools.checkCredentials);
 
-it('should lookup a dataset entry', async () => {
-  const output = await tools.runAsync(
-    `node lookupEntry.js ${projectId} ${datasetId}`,
-    cwd
-  );
-  const expectedLinkedResource = `//bigquery.googleapis.com/projects/${projectId}/datasets/${datasetId}`;
-  assert.ok(output.includes(expectedLinkedResource));
+describe('lookupEntry lookup', () => {
+  it('should lookup a dataset entry', async () => {
+    const projectId = process.env.GCLOUD_PUBLIC_PROJECT;
+    const datasetId = process.env.GCLOUD_PUBLIC_DATASET_ID;
+    const output = await tools.runAsync(
+      `node lookupEntry.js ${projectId} ${datasetId}`,
+      cwd
+    );
+    const expectedLinkedResource = `//bigquery.googleapis.com/projects/${projectId}/datasets/${datasetId}`;
+    assert.ok(output.includes(expectedLinkedResource));
+  });
 });