From 4cef015867dd80af195236efc43b309d82c7f8af Mon Sep 17 00:00:00 2001 From: Surya Krishna Guthikonda <86280547+suryaguthikonda@users.noreply.github.com> Date: Fri, 5 Aug 2022 01:26:11 +0530 Subject: [PATCH] feat(samples): add feature values apis samples (#341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added create-featurestore-sample.js and create-featurestore-sample.test.js * feat(samples): added createFeaturestore sample (#313) * feat(samples): added remaining featurestore samples (#313) * removed the extraneous files * feat(samples): add feature-values apis samples (#313) * feat(samples): add feature values apis samples (#313) * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat(samples): update dependency featurevalues apis (#313) * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: junkourata <102059808+junkourata@users.noreply.github.com> --- README.md | 5 + samples/README.md | 90 ++++++ samples/batch-read-feature-values-sample.js | 136 ++++++++ samples/export-feature-values-sample.js | 105 ++++++ .../export-feature-values-snapshot-sample.js | 113 +++++++ samples/import-feature-values-sample.js | 108 +++++++ samples/package.json | 3 +- samples/read-feature-values-sample.js | 96 ++++++ samples/test/feature-values-samples.test.js | 306 ++++++++++++++++++ 9 files changed, 961 insertions(+), 1 deletion(-) create mode 100644 samples/batch-read-feature-values-sample.js create mode 100644 samples/export-feature-values-sample.js create mode 100644 samples/export-feature-values-snapshot-sample.js create mode 100644 samples/import-feature-values-sample.js create mode 100644 samples/read-feature-values-sample.js create mode 100644 samples/test/feature-values-samples.test.js diff --git a/README.md b/README.md index 2e4bd964..5bf17867 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-ai-platform | Sample | Source Code | Try it | | --------------------------- | --------------------------------- | ------ | | Batch-create-features-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/batch-create-features-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/batch-create-features-sample.js,samples/README.md) | +| Batch-read-feature-values-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/batch-read-feature-values-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/batch-read-feature-values-sample.js,samples/README.md) | | Cancel-batch-prediction-job | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/cancel-batch-prediction-job.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/cancel-batch-prediction-job.js,samples/README.md) | | Cancel-custom-job | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/cancel-custom-job.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/cancel-custom-job.js,samples/README.md) | | Create-batch-prediction-job-text-classification | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/create-batch-prediction-job-text-classification.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/create-batch-prediction-job-text-classification.js,samples/README.md) | @@ -121,6 +122,8 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-ai-platform | Delete-model | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/delete-model.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/delete-model.js,samples/README.md) | | Deploy-model-custom-trained-model | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/deploy-model-custom-trained-model.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/deploy-model-custom-trained-model.js,samples/README.md) | | Deploy-model | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/deploy-model.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/deploy-model.js,samples/README.md) | +| Export-feature-values-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-feature-values-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-feature-values-sample.js,samples/README.md) | +| Export-feature-values-snapshot-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-feature-values-snapshot-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-feature-values-snapshot-sample.js,samples/README.md) | | Export-model-tabular-classification | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-model-tabular-classification.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-model-tabular-classification.js,samples/README.md) | | Export-model | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-model.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-model.js,samples/README.md) | | Get-batch-prediction-job | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/get-batch-prediction-job.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/get-batch-prediction-job.js,samples/README.md) | @@ -152,6 +155,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-ai-platform | Import-data-video-classification | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/import-data-video-classification.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/import-data-video-classification.js,samples/README.md) | | Import-data-video-object-tracking | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/import-data-video-object-tracking.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/import-data-video-object-tracking.js,samples/README.md) | | Import-data | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/import-data.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/import-data.js,samples/README.md) | +| Import-feature-values-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/import-feature-values-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/import-feature-values-sample.js,samples/README.md) | | List-endpoints | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/list-endpoints.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/list-endpoints.js,samples/README.md) | | List-entity-types-async-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/list-entity-types-async-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/list-entity-types-async-sample.js,samples/README.md) | | List-entity-types-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/list-entity-types-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/list-entity-types-sample.js,samples/README.md) | @@ -172,6 +176,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-ai-platform | Predict-text-entity-extraction | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/predict-text-entity-extraction.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/predict-text-entity-extraction.js,samples/README.md) | | Predict-text-sentiment-analysis | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/predict-text-sentiment-analysis.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/predict-text-sentiment-analysis.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | +| Read-feature-values-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/read-feature-values-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/read-feature-values-sample.js,samples/README.md) | | Search-features-async-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/search-features-async-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/search-features-async-sample.js,samples/README.md) | | Search-features-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/search-features-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/search-features-sample.js,samples/README.md) | | Search-features-stream-sample | [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/search-features-stream-sample.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/search-features-stream-sample.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index 14a4605b..3099f795 100644 --- a/samples/README.md +++ b/samples/README.md @@ -16,6 +16,7 @@ machine learning development lifecycle. * [Before you begin](#before-you-begin) * [Samples](#samples) * [Batch-create-features-sample](#batch-create-features-sample) + * [Batch-read-feature-values-sample](#batch-read-feature-values-sample) * [Cancel-batch-prediction-job](#cancel-batch-prediction-job) * [Cancel-custom-job](#cancel-custom-job) * [Create-batch-prediction-job-text-classification](#create-batch-prediction-job-text-classification) @@ -59,6 +60,8 @@ machine learning development lifecycle. * [Delete-model](#delete-model) * [Deploy-model-custom-trained-model](#deploy-model-custom-trained-model) * [Deploy-model](#deploy-model) + * [Export-feature-values-sample](#export-feature-values-sample) + * [Export-feature-values-snapshot-sample](#export-feature-values-snapshot-sample) * [Export-model-tabular-classification](#export-model-tabular-classification) * [Export-model](#export-model) * [Get-batch-prediction-job](#get-batch-prediction-job) @@ -90,6 +93,7 @@ machine learning development lifecycle. * [Import-data-video-classification](#import-data-video-classification) * [Import-data-video-object-tracking](#import-data-video-object-tracking) * [Import-data](#import-data) + * [Import-feature-values-sample](#import-feature-values-sample) * [List-endpoints](#list-endpoints) * [List-entity-types-async-sample](#list-entity-types-async-sample) * [List-entity-types-sample](#list-entity-types-sample) @@ -110,6 +114,7 @@ machine learning development lifecycle. * [Predict-text-entity-extraction](#predict-text-entity-extraction) * [Predict-text-sentiment-analysis](#predict-text-sentiment-analysis) * [Quickstart](#quickstart) + * [Read-feature-values-sample](#read-feature-values-sample) * [Search-features-async-sample](#search-features-async-sample) * [Search-features-sample](#search-features-sample) * [Search-features-stream-sample](#search-features-stream-sample) @@ -153,6 +158,23 @@ __Usage:__ +### Batch-read-feature-values-sample + +View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/batch-read-feature-values-sample.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/batch-read-feature-values-sample.js,samples/README.md) + +__Usage:__ + + +`node samples/batch-read-feature-values-sample.js` + + +----- + + + + ### Cancel-batch-prediction-job View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/cancel-batch-prediction-job.js). @@ -884,6 +906,40 @@ __Usage:__ +### Export-feature-values-sample + +View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-feature-values-sample.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-feature-values-sample.js,samples/README.md) + +__Usage:__ + + +`node samples/export-feature-values-sample.js` + + +----- + + + + +### Export-feature-values-snapshot-sample + +View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-feature-values-snapshot-sample.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/export-feature-values-snapshot-sample.js,samples/README.md) + +__Usage:__ + + +`node samples/export-feature-values-snapshot-sample.js` + + +----- + + + + ### Export-model-tabular-classification View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/export-model-tabular-classification.js). @@ -1411,6 +1467,23 @@ __Usage:__ +### Import-feature-values-sample + +View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/import-feature-values-sample.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/import-feature-values-sample.js,samples/README.md) + +__Usage:__ + + +`node samples/import-feature-values-sample.js` + + +----- + + + + ### List-endpoints View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/list-endpoints.js). @@ -1751,6 +1824,23 @@ __Usage:__ +### Read-feature-values-sample + +View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/read-feature-values-sample.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-ai-platform&page=editor&open_in_editor=samples/read-feature-values-sample.js,samples/README.md) + +__Usage:__ + + +`node samples/read-feature-values-sample.js` + + +----- + + + + ### Search-features-async-sample View the [source code](https://github.com/googleapis/nodejs-ai-platform/blob/main/samples/search-features-async-sample.js). diff --git a/samples/batch-read-feature-values-sample.js b/samples/batch-read-feature-values-sample.js new file mode 100644 index 00000000..75a19893 --- /dev/null +++ b/samples/batch-read-feature-values-sample.js @@ -0,0 +1,136 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Batch reads Feature values from a Featurestore. + * See https://cloud.google.com/vertex-ai/docs/featurestore/setup before running + * the code snippet + */ + +'use strict'; + +async function main( + project, + featurestoreId, + inputCsvFile, + destinationTableUri, + location = 'us-central1', + apiEndpoint = 'us-central1-aiplatform.googleapis.com', + timeout = 300000 +) { + // [START aiplatform_batch_read_feature_values_sample] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const project = 'YOUR_PROJECT_ID'; + // const featurestoreId = 'YOUR_FEATURESTORE_ID'; + // const inputCsvFile = 'YOUR_INPUT_CSV_FILE_URI'; + // const destinationTableUri = 'YOUR_BQ_DESTINATION_TABLE_URI'; + // const location = 'YOUR_PROJECT_LOCATION'; + // const apiEndpoint = 'YOUR_API_ENDPOINT'; + // const timeout = ; + + // Imports the Google Cloud Featurestore Service Client library + const {FeaturestoreServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: apiEndpoint, + }; + + // Instantiates a client + const featurestoreServiceClient = new FeaturestoreServiceClient( + clientOptions + ); + + async function batchReadFeatureValues() { + // Configure the featurestoreId resource + const featurestore = `projects/${project}/locations/${location}/featurestores/${featurestoreId}`; + const csvReadInstances = { + gcsSource: { + uris: [inputCsvFile], + }, + }; + + const destination = { + bigqueryDestination: { + // # Output to BigQuery table created earlier + outputUri: destinationTableUri, + }, + }; + + const usersFeatureSelector = { + idMatcher: { + ids: [ + // features, use "*" if you want to select all features within this entity type + 'age', + 'gender', + 'liked_genres', + ], + }, + }; + + const usersEntityTypeSpec = { + // Read the 'age', 'gender' and 'liked_genres' features from the 'perm_users' entity + entityTypeId: 'perm_users', + featureSelector: usersFeatureSelector, + }; + + const moviesFeatureSelector = { + idMatcher: { + ids: ['*'], + }, + }; + + const moviesEntityTypeSpec = { + // Read the all features from the 'perm_movies' entity + entityTypeId: 'perm_movies', + featureSelector: moviesFeatureSelector, + }; + + const entityTypeSpecs = [usersEntityTypeSpec, moviesEntityTypeSpec]; + + // Construct request + const request = { + featurestore: featurestore, + csvReadInstances: csvReadInstances, + destination: destination, + entityTypeSpecs: entityTypeSpecs, + }; + + // Batch Read Feature Values Request + const [operation] = await featurestoreServiceClient.batchReadFeatureValues( + request, + {timeout: Number(timeout)} + ); + const [response] = await operation.promise(); + + console.log('Batch read feature values response'); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + batchReadFeatureValues(); + // [END aiplatform_batch_read_feature_values_sample] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/samples/export-feature-values-sample.js b/samples/export-feature-values-sample.js new file mode 100644 index 00000000..04d9c34c --- /dev/null +++ b/samples/export-feature-values-sample.js @@ -0,0 +1,105 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Exports Feature values from all the entities of a target EntityType. + * See https://cloud.google.com/vertex-ai/docs/featurestore/setup before running + * the code snippet + */ + +'use strict'; + +async function main( + project, + featurestoreId, + entityTypeId, + destinationTableUri, + location = 'us-central1', + apiEndpoint = 'us-central1-aiplatform.googleapis.com', + timeout = 300000 +) { + // [START aiplatform_export_feature_values_sample] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const project = 'YOUR_PROJECT_ID'; + // const featurestoreId = 'YOUR_FEATURESTORE_ID'; + // const entityTypeId = 'YOUR_ENTITY_TYPE_ID'; + // const destinationTableUri = 'YOUR_BQ_DESTINATION_TABLE_URI'; + // const location = 'YOUR_PROJECT_LOCATION'; + // const apiEndpoint = 'YOUR_API_ENDPOINT'; + // const timeout = ; + + // Imports the Google Cloud Featurestore Service Client library + const {FeaturestoreServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: apiEndpoint, + }; + + // Instantiates a client + const featurestoreServiceClient = new FeaturestoreServiceClient( + clientOptions + ); + + async function exportFeatureValues() { + // Configure the entityType resource + const entityType = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId}`; + + const destination = { + bigqueryDestination: { + // # Output to BigQuery table created earlier + outputUri: destinationTableUri, + }, + }; + + const featureSelector = { + idMatcher: { + ids: ['age', 'gender', 'liked_genres'], + }, + }; + + const request = { + entityType: entityType, + destination: destination, + featureSelector: featureSelector, + fullExport: {}, + }; + + // Export Feature Values Request + const [operation] = await featurestoreServiceClient.exportFeatureValues( + request, + {timeout: Number(timeout)} + ); + const [response] = await operation.promise(); + + console.log('Export feature values response'); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + exportFeatureValues(); + // [END aiplatform_export_feature_values_sample] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/samples/export-feature-values-snapshot-sample.js b/samples/export-feature-values-snapshot-sample.js new file mode 100644 index 00000000..f527bfb1 --- /dev/null +++ b/samples/export-feature-values-snapshot-sample.js @@ -0,0 +1,113 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Exports Feature values with snapshot from all the entities of a target EntityType. + * See https://cloud.google.com/vertex-ai/docs/featurestore/setup before running + * the code snippet + */ + +'use strict'; + +async function main( + project, + featurestoreId, + entityTypeId, + destinationTableUri, + timestamp, + location = 'us-central1', + apiEndpoint = 'us-central1-aiplatform.googleapis.com', + timeout = 300000 +) { + // [START aiplatform_export_feature_values_snapshot_sample] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const project = 'YOUR_PROJECT_ID'; + // const featurestoreId = 'YOUR_FEATURESTORE_ID'; + // const entityTypeId = 'YOUR_ENTITY_TYPE_ID'; + // const destinationTableUri = 'YOUR_BQ_DESTINATION_TABLE_URI'; + // const timestamp = ; + // const location = 'YOUR_PROJECT_LOCATION'; + // const apiEndpoint = 'YOUR_API_ENDPOINT'; + // const timeout = ; + + // Imports the Google Cloud Featurestore Service Client library + const {FeaturestoreServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: apiEndpoint, + }; + + // Instantiates a client + const featurestoreServiceClient = new FeaturestoreServiceClient( + clientOptions + ); + + async function exportFeatureValuesSnapshot() { + // Configure the entityType resource + const entityType = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId}`; + + const destination = { + bigqueryDestination: { + // # Output to BigQuery table created earlier + outputUri: destinationTableUri, + }, + }; + + const featureSelector = { + idMatcher: { + ids: ['age', 'gender', 'liked_genres'], + }, + }; + + const snapshotExport = { + startTime: { + seconds: Number(timestamp), + }, + }; + + const request = { + entityType: entityType, + destination: destination, + featureSelector: featureSelector, + snapshotExport: snapshotExport, + }; + + // Export Feature Values Request + const [operation] = await featurestoreServiceClient.exportFeatureValues( + request, + {timeout: Number(timeout)} + ); + const [response] = await operation.promise(); + + console.log('Export feature values snapshot response'); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + exportFeatureValuesSnapshot(); + // [END aiplatform_export_feature_values_snapshot_sample] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/samples/import-feature-values-sample.js b/samples/import-feature-values-sample.js new file mode 100644 index 00000000..01a174bf --- /dev/null +++ b/samples/import-feature-values-sample.js @@ -0,0 +1,108 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Imports Feature values into the Featurestore from a source storage. + * See https://cloud.google.com/vertex-ai/docs/featurestore/setup before running + * the code snippet + */ + +'use strict'; + +async function main( + project, + featurestoreId, + entityTypeId, + avroGcsUri, + entityIdField, + featureTimeField, + workerCount = 2, + location = 'us-central1', + apiEndpoint = 'us-central1-aiplatform.googleapis.com', + timeout = 300000 +) { + // [START aiplatform_import_feature_values_sample] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const project = 'YOUR_PROJECT_ID'; + // const featurestoreId = 'YOUR_FEATURESTORE_ID'; + // const entityTypeId = 'YOUR_ENTITY_TYPE_ID'; + // const avroGcsUri = 'AVRO_FILE_IN_THE_GCS_URI'; + // const entityIdField = 'ENTITY_ID_FIELD_IN_AVRO'; + // const featureTimeField = 'TIMESTAMP_FIELD_IN_AVRO'; + // const workerCount = ; + // const location = 'YOUR_PROJECT_LOCATION'; + // const apiEndpoint = 'YOUR_API_ENDPOINT'; + // const timeout = ; + + // Imports the Google Cloud Featurestore Service Client library + const {FeaturestoreServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: apiEndpoint, + }; + + // Instantiates a client + const featurestoreServiceClient = new FeaturestoreServiceClient( + clientOptions + ); + + async function importFeatureValues() { + // Configure the entityType resource + const entityType = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId}`; + + const avroSource = { + gcsSource: { + uris: [avroGcsUri], + }, + }; + + const featureSpecs = [{id: 'age'}, {id: 'gender'}, {id: 'liked_genres'}]; + + const request = { + entityType: entityType, + avroSource: avroSource, + entityIdField: entityIdField, + featureSpecs: featureSpecs, + featureTimeField: featureTimeField, + workerCount: Number(workerCount), + }; + + // Import Feature Values Request + const [operation] = await featurestoreServiceClient.importFeatureValues( + request, + {timeout: Number(timeout)} + ); + const [response] = await operation.promise(); + + console.log('Import feature values response'); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + importFeatureValues(); + // [END aiplatform_import_feature_values_sample] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/samples/package.json b/samples/package.json index 51aeadb2..40abea41 100644 --- a/samples/package.json +++ b/samples/package.json @@ -14,7 +14,8 @@ }, "dependencies": { "@google-cloud/aiplatform": "^2.1.0", - "@google-cloud/storage": "^5.5.0" + "@google-cloud/storage": "^5.5.0", + "@google-cloud/bigquery": "^6.0.0" }, "devDependencies": { "chai": "^4.2.0", diff --git a/samples/read-feature-values-sample.js b/samples/read-feature-values-sample.js new file mode 100644 index 00000000..df4f5a37 --- /dev/null +++ b/samples/read-feature-values-sample.js @@ -0,0 +1,96 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Reads Feature values of a specific entity of an EntityType. + * See https://cloud.google.com/vertex-ai/docs/featurestore/setup before running + * the code snippet + */ + +'use strict'; + +async function main( + project, + featurestoreId, + entityTypeId, + entityId, + location = 'us-central1', + apiEndpoint = 'us-central1-aiplatform.googleapis.com', + timeout = 300000 +) { + // [START aiplatform_read_feature_values_sample] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const project = 'YOUR_PROJECT_ID'; + // const featurestoreId = 'YOUR_FEATURESTORE_ID'; + // const entityTypeId = 'YOUR_ENTITY_TYPE_ID'; + // const entityId = 'ENTITY_ID_TO_SERVE'; + // const location = 'YOUR_PROJECT_LOCATION'; + // const apiEndpoint = 'YOUR_API_ENDPOINT'; + // const timeout = ; + + // Imports the Google Cloud Featurestore Service Client library + const {FeaturestoreOnlineServingServiceClient} = + require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: apiEndpoint, + }; + + // Instantiates a client + const featurestoreOnlineServingServiceClient = + new FeaturestoreOnlineServingServiceClient(clientOptions); + + async function readFeatureValues() { + // Configure the entityType resource + const entityType = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId}`; + + const featureSelector = { + idMatcher: { + ids: ['age', 'gender', 'liked_genres'], + }, + }; + + const request = { + entityType: entityType, + entityId: entityId, + featureSelector: featureSelector, + }; + + // Read Feature Values Request + const [response] = + await featurestoreOnlineServingServiceClient.readFeatureValues(request, { + timeout: Number(timeout), + }); + + console.log('Read feature values response'); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + readFeatureValues(); + // [END aiplatform_read_feature_values_sample] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/samples/test/feature-values-samples.test.js b/samples/test/feature-values-samples.test.js new file mode 100644 index 00000000..17ce0946 --- /dev/null +++ b/samples/test/feature-values-samples.test.js @@ -0,0 +1,306 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {BigQuery} = require('@google-cloud/bigquery'); +const {FeaturestoreServiceClient} = require('@google-cloud/aiplatform').v1; +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); +const uuid = require('uuid').v4; +const cp = require('child_process'); +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const project = process.env.CAIP_PROJECT_ID; +const datasetName = `movie_predictions_nodejs_${uuid() + .replace(/-/g, '_') + .slice(10, 20)}`; +const batchReadTableName = 'batch_serving_table'; +const exportTableName = 'export_table'; +const exportSnapshotTableName = 'export_snapshot_table'; + +const featurestoreId = `featurestore_sample_${uuid() + .replace(/-/g, '_') + .slice(10, 20)}`; +const fixedNodeCount = 1; + +const entityTypeId1 = 'perm_users'; +const entityTypeDescription1 = 'Users Entity'; +const entityTypeId2 = 'perm_movies'; +const entityTypeDescription2 = 'Movies Entity'; +const entityId = 'alice'; + +const avroGcsUri1 = + 'gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/users.avro'; +const entityIdField1 = 'user_id'; + +const avroGcsUri2 = + 'gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movies.avro'; +const entityIdField2 = 'movie_id'; + +const featureTimeField = 'update_time'; +const workerCount = 2; +const batchReadDestinationTableUri = `bq://${project}.${datasetName}.${batchReadTableName}`; +const inputCsvFile = + 'gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction_perm.csv'; + +const timestamp = 1629493102; +const exportDestinationTableUri = `bq://${project}.${datasetName}.${exportTableName}`; +const exportSnapshotDestinationTableUri = `bq://${project}.${datasetName}.${exportSnapshotTableName}`; + +const location = 'us-central1'; +const apiEndpoint = 'us-central1-aiplatform.googleapis.com'; + +// Instantiates a featurestore and bigquery clients +const featurestoreServiceClient = new FeaturestoreServiceClient({ + apiEndpoint: apiEndpoint, +}); +const bigqueryClient = new BigQuery({projectId: project}); + +const createEntityType = async (entityTypeId, entityTypeDescription) => { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}/featurestores/${featurestoreId}`; + + const entityType = { + description: entityTypeDescription, + }; + + const request = { + parent: parent, + entityTypeId: entityTypeId, + entityType: entityType, + }; + + // CreateEntityType request + const [operation] = await featurestoreServiceClient.createEntityType( + request, + {timeout: 300000} + ); + await operation.promise(); +}; + +const createPermUsersFeatures = async () => { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId1}`; + + const ageFeature = { + valueType: 'INT64', + description: 'User age', + }; + + const ageFeatureRequest = { + feature: ageFeature, + featureId: 'age', + }; + + const genderFeature = { + valueType: 'STRING', + description: 'User gender', + }; + + const genderFeatureRequest = { + feature: genderFeature, + featureId: 'gender', + }; + + const likedGenresFeature = { + valueType: 'STRING_ARRAY', + description: 'An array of genres that this user liked', + }; + + const likedGenresFeatureRequest = { + feature: likedGenresFeature, + featureId: 'liked_genres', + }; + + const requests = [ + ageFeatureRequest, + genderFeatureRequest, + likedGenresFeatureRequest, + ]; + + const request = { + parent: parent, + requests: requests, + }; + + // Batch Create Features request + const [operation] = await featurestoreServiceClient.batchCreateFeatures( + request, + {timeout: 300000} + ); + await operation.promise(); +}; + +const createPermMoviesFeatures = async () => { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId2}`; + + const titleFeatureRequest = { + feature: { + valueType: 'STRING', + description: 'The title of the movie', + }, + featureId: 'title', + }; + + const genresFeatureRequest = { + feature: { + valueType: 'STRING', + description: 'The genres of the movie', + }, + featureId: 'genres', + }; + + const averageRatingFeatureRequest = { + feature: { + valueType: 'DOUBLE', + description: 'The average rating for the movie, range is [1.0-5.0]', + }, + featureId: 'average_rating', + }; + + const requests = [ + titleFeatureRequest, + genresFeatureRequest, + averageRatingFeatureRequest, + ]; + + const request = { + parent: parent, + requests: requests, + }; + + // Batch Create Features request + const [operation] = await featurestoreServiceClient.batchCreateFeatures( + request, + {timeout: 300000} + ); + await operation.promise(); +}; + +const importPermMoviesFeatures = async () => { + // Configure the entityType resource + const entityType = `projects/${project}/locations/${location}/featurestores/${featurestoreId}/entityTypes/${entityTypeId2}`; + + const avroSource = { + gcsSource: { + uris: [avroGcsUri2], + }, + }; + + const featureSpecs = [{id: 'title'}, {id: 'genres'}, {id: 'average_rating'}]; + + const request = { + entityType: entityType, + avroSource: avroSource, + entityIdField: entityIdField2, + featureSpecs: featureSpecs, + featureTimeField: featureTimeField, + workerCount: workerCount, + }; + + // Import Feature Values Request + const [operation] = await featurestoreServiceClient.importFeatureValues( + request, + {timeout: 300000} + ); + await operation.promise(); +}; + +const deleteFeaturestore = async () => { + // Configure the name resource + const name = `projects/${project}/locations/${location}/featurestores/${featurestoreId}`; + + const request = { + name: name, + force: true, + }; + + // Delete Featurestore request + const [operation] = await featurestoreServiceClient.deleteFeaturestore( + request, + {timeout: 60000} + ); + await operation.promise(); +}; + +describe('AI platform feature values apis', async function () { + this.retries(2); + before('should create the BigQuery Dataset', async () => { + await bigqueryClient.createDataset(datasetName, location); + }); + before('should create the featurestore', async () => { + execSync( + `node ./create-featurestore-fixed-nodes-sample.js ${project} ${featurestoreId} ${fixedNodeCount} ${location} ${apiEndpoint}` + ); + }); + before('should create the perm_users entity type', async () => { + await createEntityType(entityTypeId1, entityTypeDescription1); + }); + before('should create the perm_movies entity type', async () => { + await createEntityType(entityTypeId2, entityTypeDescription2); + }); + before('should create the perm_movies batch features', async () => { + await createPermMoviesFeatures(); + }); + before('should create the perm_users batch features', async () => { + await createPermUsersFeatures(); + }); + before('should import perm_movies feature values', async () => { + await importPermMoviesFeatures(); + }); + it('should import feature values', async () => { + const stdout = execSync( + `node ./import-feature-values-sample.js ${project} ${featurestoreId} ${entityTypeId1} "${avroGcsUri1}" ${entityIdField1} ${featureTimeField} ${workerCount} ${location} ${apiEndpoint}` + ); + assert.match(stdout, /Import feature values response/); + }); + it('should batch read feature values', async () => { + const stdout = execSync( + `node ./batch-read-feature-values-sample.js ${project} ${featurestoreId} ${inputCsvFile} "${batchReadDestinationTableUri}" ${location} ${apiEndpoint}` + ); + assert.match(stdout, /Batch read feature values response/); + }); + it('should read feature values', async () => { + const stdout = execSync( + `node ./read-feature-values-sample.js ${project} ${featurestoreId} ${entityTypeId1} ${entityId} ${location} ${apiEndpoint}` + ); + assert.match(stdout, /Read feature values response/); + }); + it('should export feature values', async () => { + const stdout = execSync( + `node ./export-feature-values-sample.js ${project} ${featurestoreId} ${entityTypeId1} "${exportDestinationTableUri}" ${location} ${apiEndpoint}` + ); + assert.match(stdout, /Export feature values response/); + }); + it('should export feature values using snapshot', async () => { + const stdout = execSync( + `node ./export-feature-values-snapshot-sample.js ${project} ${featurestoreId} ${entityTypeId1} "${exportSnapshotDestinationTableUri}" ${timestamp} ${location} ${apiEndpoint}` + ); + assert.match(stdout, /Export feature values snapshot response/); + }); + after('should delete the created featurestore', async () => { + await deleteFeaturestore(); + }); + + after('should delete the created dataset', async () => { + // Create a reference to the existing dataset + const dataset = bigqueryClient.dataset(datasetName); + // Delete the dataset and its contents + await dataset.delete({force: true}); + }); +});