feat: add monitoring_config to EntityType in aiplatform v1 entity_type.proto

google/cloud/aiplatform_v1/__init__.py

@@ -131,6 +131,7 @@
 from .types.feature_selector import FeatureSelector
 from .types.feature_selector import IdMatcher
 from .types.featurestore import Featurestore
+from .types.featurestore_monitoring import FeaturestoreMonitoringConfig
 from .types.featurestore_online_service import FeatureValue
 from .types.featurestore_online_service import FeatureValueList
 from .types.featurestore_online_service import ReadFeatureValuesRequest
@@ -672,6 +673,7 @@
     "FeatureValueDestination",
     "FeatureValueList",
     "Featurestore",
+    "FeaturestoreMonitoringConfig",
     "FeaturestoreOnlineServingServiceClient",
     "FeaturestoreServiceClient",
     "FilterSplit",

google/cloud/aiplatform_v1/services/featurestore_service/async_client.py

@@ -41,6 +41,7 @@
 from google.cloud.aiplatform_v1.types import feature as gca_feature
 from google.cloud.aiplatform_v1.types import featurestore
 from google.cloud.aiplatform_v1.types import featurestore as gca_featurestore
+from google.cloud.aiplatform_v1.types import featurestore_monitoring
 from google.cloud.aiplatform_v1.types import featurestore_service
 from google.cloud.aiplatform_v1.types import operation as gca_operation
 from google.protobuf import empty_pb2  # type: ignore
@@ -1202,7 +1203,12 @@ def sample_update_entity_type():
                 -  ``description``
                 -  ``labels``
                 -  ``monitoring_config.snapshot_analysis.disabled``
-                -  ``monitoring_config.snapshot_analysis.monitoring_interval``
+                -  ``monitoring_config.snapshot_analysis.monitoring_interval_days``
+                -  ``monitoring_config.snapshot_analysis.staleness_days``
+                -  ``monitoring_config.import_features_analysis.state``
+                -  ``monitoring_config.import_features_analysis.anomaly_detection_baseline``
+                -  ``monitoring_config.numerical_threshold_config.value``
+                -  ``monitoring_config.categorical_threshold_config.value``
 
                 This corresponds to the ``update_mask`` field
                 on the ``request`` instance; if ``request`` is provided, this
@@ -1912,8 +1918,7 @@ def sample_update_feature():
 
                 -  ``description``
                 -  ``labels``
-                -  ``monitoring_config.snapshot_analysis.disabled``
-                -  ``monitoring_config.snapshot_analysis.monitoring_interval``
+                -  ``disable_monitoring``
 
                 This corresponds to the ``update_mask`` field
                 on the ``request`` instance; if ``request`` is provided, this

google/cloud/aiplatform_v1/services/featurestore_service/client.py

@@ -44,6 +44,7 @@
 from google.cloud.aiplatform_v1.types import feature as gca_feature
 from google.cloud.aiplatform_v1.types import featurestore
 from google.cloud.aiplatform_v1.types import featurestore as gca_featurestore
+from google.cloud.aiplatform_v1.types import featurestore_monitoring
 from google.cloud.aiplatform_v1.types import featurestore_service
 from google.cloud.aiplatform_v1.types import operation as gca_operation
 from google.protobuf import empty_pb2  # type: ignore
@@ -1435,7 +1436,12 @@ def sample_update_entity_type():
                 -  ``description``
                 -  ``labels``
                 -  ``monitoring_config.snapshot_analysis.disabled``
-                -  ``monitoring_config.snapshot_analysis.monitoring_interval``
+                -  ``monitoring_config.snapshot_analysis.monitoring_interval_days``
+                -  ``monitoring_config.snapshot_analysis.staleness_days``
+                -  ``monitoring_config.import_features_analysis.state``
+                -  ``monitoring_config.import_features_analysis.anomaly_detection_baseline``
+                -  ``monitoring_config.numerical_threshold_config.value``
+                -  ``monitoring_config.categorical_threshold_config.value``
 
                 This corresponds to the ``update_mask`` field
                 on the ``request`` instance; if ``request`` is provided, this
@@ -2145,8 +2151,7 @@ def sample_update_feature():
 
                 -  ``description``
                 -  ``labels``
-                -  ``monitoring_config.snapshot_analysis.disabled``
-                -  ``monitoring_config.snapshot_analysis.monitoring_interval``
+                -  ``disable_monitoring``
 
                 This corresponds to the ``update_mask`` field
                 on the ``request`` instance; if ``request`` is provided, this

google/cloud/aiplatform_v1/types/__init__.py

@@ -110,6 +110,7 @@
     IdMatcher,
 )
 from .featurestore import Featurestore
+from .featurestore_monitoring import FeaturestoreMonitoringConfig
 from .featurestore_online_service import (
     FeatureValue,
     FeatureValueList,
@@ -585,6 +586,7 @@
     "FeatureSelector",
     "IdMatcher",
     "Featurestore",
+    "FeaturestoreMonitoringConfig",
     "FeatureValue",
     "FeatureValueList",
     "ReadFeatureValuesRequest",

google/cloud/aiplatform_v1/types/entity_type.py

@@ -15,6 +15,7 @@
 #
 import proto  # type: ignore
 
+from google.cloud.aiplatform_v1.types import featurestore_monitoring
 from google.protobuf import timestamp_pb2  # type: ignore
 
 
@@ -65,6 +66,16 @@ class EntityType(proto.Message):
             Optional. Used to perform a consistent
             read-modify-write updates. If not set, a blind
             "overwrite" update happens.
+        monitoring_config (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig):
+            Optional. The default monitoring configuration for all
+            Features with value type
+            ([Feature.ValueType][google.cloud.aiplatform.v1.Feature.ValueType])
+            BOOL, STRING, DOUBLE or INT64 under this EntityType.
+
+            If this is populated with
+            [FeaturestoreMonitoringConfig.monitoring_interval]
+            specified, snapshot analysis monitoring is enabled.
+            Otherwise, snapshot analysis monitoring is disabled.
     """
 
     name = proto.Field(proto.STRING, number=1,)
@@ -73,6 +84,11 @@ class EntityType(proto.Message):
     update_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,)
     labels = proto.MapField(proto.STRING, proto.STRING, number=6,)
     etag = proto.Field(proto.STRING, number=7,)
+    monitoring_config = proto.Field(
+        proto.MESSAGE,
+        number=8,
+        message=featurestore_monitoring.FeaturestoreMonitoringConfig,
+    )
 
 
 __all__ = tuple(sorted(__protobuf__.manifest))

google/cloud/aiplatform_v1/types/feature.py

@@ -15,6 +15,7 @@
 #
 import proto  # type: ignore
 
+from google.cloud.aiplatform_v1.types import feature_monitoring_stats
 from google.protobuf import timestamp_pb2  # type: ignore
 
 
@@ -66,6 +67,18 @@ class Feature(proto.Message):
             Used to perform a consistent
             read-modify-write updates. If not set, a blind
             "overwrite" update happens.
+        disable_monitoring (bool):
+            Optional. If not set, use the monitoring_config defined for
+            the EntityType this Feature belongs to. Only Features with
+            type
+            ([Feature.ValueType][google.cloud.aiplatform.v1.Feature.ValueType])
+            BOOL, STRING, DOUBLE or INT64 can enable monitoring.
+
+            If set to true, all types of data monitoring are disabled
+            despite the config on EntityType.
+        monitoring_stats_anomalies (Sequence[google.cloud.aiplatform_v1.types.Feature.MonitoringStatsAnomaly]):
+            Output only. The list of historical stats and
+            anomalies with specified objectives.
     """
 
     class ValueType(proto.Enum):
@@ -81,13 +94,53 @@ class ValueType(proto.Enum):
         STRING_ARRAY = 12
         BYTES = 13
 
+    class MonitoringStatsAnomaly(proto.Message):
+        r"""A list of historical [Snapshot
+        Analysis][FeaturestoreMonitoringConfig.SnapshotAnalysis] or [Import
+        Feature Analysis]
+        [FeaturestoreMonitoringConfig.ImportFeatureAnalysis] stats requested
+        by user, sorted by
+        [FeatureStatsAnomaly.start_time][google.cloud.aiplatform.v1.FeatureStatsAnomaly.start_time]
+        descending.
+
+        Attributes:
+            objective (google.cloud.aiplatform_v1.types.Feature.MonitoringStatsAnomaly.Objective):
+                Output only. The objective for each stats.
+            feature_stats_anomaly (google.cloud.aiplatform_v1.types.FeatureStatsAnomaly):
+                Output only. The stats and anomalies
+                generated at specific timestamp.
+        """
+
+        class Objective(proto.Enum):
+            r"""If the objective in the request is both
+            Import Feature Analysis and Snapshot Analysis, this objective
+            could be one of them. Otherwise, this objective should be the
+            same as the objective in the request.
+            """
+            OBJECTIVE_UNSPECIFIED = 0
+            IMPORT_FEATURE_ANALYSIS = 1
+            SNAPSHOT_ANALYSIS = 2
+
+        objective = proto.Field(
+            proto.ENUM, number=1, enum="Feature.MonitoringStatsAnomaly.Objective",
+        )
+        feature_stats_anomaly = proto.Field(
+            proto.MESSAGE,
+            number=2,
+            message=feature_monitoring_stats.FeatureStatsAnomaly,
+        )
+
     name = proto.Field(proto.STRING, number=1,)
     description = proto.Field(proto.STRING, number=2,)
     value_type = proto.Field(proto.ENUM, number=3, enum=ValueType,)
     create_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,)
     update_time = proto.Field(proto.MESSAGE, number=5, message=timestamp_pb2.Timestamp,)
     labels = proto.MapField(proto.STRING, proto.STRING, number=6,)
     etag = proto.Field(proto.STRING, number=7,)
+    disable_monitoring = proto.Field(proto.BOOL, number=12,)
+    monitoring_stats_anomalies = proto.RepeatedField(
+        proto.MESSAGE, number=11, message=MonitoringStatsAnomaly,
+    )
 
 
 __all__ = tuple(sorted(__protobuf__.manifest))

google/cloud/aiplatform_v1/types/featurestore_monitoring.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import proto  # type: ignore
+
+
+__protobuf__ = proto.module(
+    package="google.cloud.aiplatform.v1", manifest={"FeaturestoreMonitoringConfig",},
+)
+
+
+class FeaturestoreMonitoringConfig(proto.Message):
+    r"""Configuration of how features in Featurestore are monitored.
+
+    Attributes:
+        snapshot_analysis (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.SnapshotAnalysis):
+            The config for Snapshot Analysis Based
+            Feature Monitoring.
+        import_features_analysis (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis):
+            The config for ImportFeatures Analysis Based
+            Feature Monitoring.
+        numerical_threshold_config (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.ThresholdConfig):
+            Threshold for numerical features of anomaly detection. This
+            is shared by all objectives of Featurestore Monitoring for
+            numerical features (i.e. Features with type
+            ([Feature.ValueType][google.cloud.aiplatform.v1.Feature.ValueType])
+            DOUBLE or INT64).
+        categorical_threshold_config (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.ThresholdConfig):
+            Threshold for categorical features of anomaly detection.
+            This is shared by all types of Featurestore Monitoring for
+            categorical features (i.e. Features with type
+            ([Feature.ValueType][google.cloud.aiplatform.v1.Feature.ValueType])
+            BOOL or STRING).
+    """
+
+    class SnapshotAnalysis(proto.Message):
+        r"""Configuration of the Featurestore's Snapshot Analysis Based
+        Monitoring. This type of analysis generates statistics for each
+        Feature based on a snapshot of the latest feature value of each
+        entities every monitoring_interval.
+
+        Attributes:
+            disabled (bool):
+                The monitoring schedule for snapshot analysis. For
+                EntityType-level config: unset / disabled = true indicates
+                disabled by default for Features under it; otherwise by
+                default enable snapshot analysis monitoring with
+                monitoring_interval for Features under it. Feature-level
+                config: disabled = true indicates disabled regardless of the
+                EntityType-level config; unset monitoring_interval indicates
+                going with EntityType-level config; otherwise run snapshot
+                analysis monitoring with monitoring_interval regardless of
+                the EntityType-level config. Explicitly Disable the snapshot
+                analysis based monitoring.
+            monitoring_interval_days (int):
+                Configuration of the snapshot analysis based monitoring
+                pipeline running interval. The value indicates number of
+                days. If both
+                [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days][google.cloud.aiplatform.v1.FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days]
+                and
+                [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval][]
+                are set when creating/updating EntityTypes/Features,
+                [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days][google.cloud.aiplatform.v1.FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days]
+                will be used.
+            staleness_days (int):
+                Customized export features time window for
+                snapshot analysis. Unit is one day. Default
+                value is 3 weeks. Minimum value is 1 day.
+                Maximum value is 4000 days.
+        """
+
+        disabled = proto.Field(proto.BOOL, number=1,)
+        monitoring_interval_days = proto.Field(proto.INT32, number=3,)
+        staleness_days = proto.Field(proto.INT32, number=4,)
+
+    class ImportFeaturesAnalysis(proto.Message):
+        r"""Configuration of the Featurestore's ImportFeature Analysis Based
+        Monitoring. This type of analysis generates statistics for values of
+        each Feature imported by every [ImportFeatureValues][] operation.
+
+        Attributes:
+            state (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State):
+                Whether to enable / disable / inherite
+                default hebavior for import features analysis.
+            anomaly_detection_baseline (google.cloud.aiplatform_v1.types.FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline):
+                The baseline used to do anomaly detection for
+                the statistics generated by import features
+                analysis.
+        """
+
+        class State(proto.Enum):
+            r"""The state defines whether to enable ImportFeature analysis."""
+            STATE_UNSPECIFIED = 0
+            DEFAULT = 1
+            ENABLED = 2
+            DISABLED = 3
+
+        class Baseline(proto.Enum):
+            r"""Defines the baseline to do anomaly detection for feature values
+            imported by each [ImportFeatureValues][] operation.
+            """
+            BASELINE_UNSPECIFIED = 0
+            LATEST_STATS = 1
+            MOST_RECENT_SNAPSHOT_STATS = 2
+            PREVIOUS_IMPORT_FEATURES_STATS = 3
+
+        state = proto.Field(
+            proto.ENUM,
+            number=1,
+            enum="FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.State",
+        )
+        anomaly_detection_baseline = proto.Field(
+            proto.ENUM,
+            number=2,
+            enum="FeaturestoreMonitoringConfig.ImportFeaturesAnalysis.Baseline",
+        )
+
+    class ThresholdConfig(proto.Message):
+        r"""The config for Featurestore Monitoring threshold.
+
+        .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
+
+        Attributes:
+            value (float):
+                Specify a threshold value that can trigger
+                the alert. 1. For categorical feature, the
+                distribution distance is calculated by
+                L-inifinity norm.
+                2. For numerical feature, the distribution
+                distance is calculated by Jensen–Shannon
+                divergence. Each feature must have a non-zero
+                threshold if they need to be monitored.
+                Otherwise no alert will be triggered for that
+                feature.
+
+                This field is a member of `oneof`_ ``threshold``.
+        """
+
+        value = proto.Field(proto.DOUBLE, number=1, oneof="threshold",)
+
+    snapshot_analysis = proto.Field(proto.MESSAGE, number=1, message=SnapshotAnalysis,)
+    import_features_analysis = proto.Field(
+        proto.MESSAGE, number=2, message=ImportFeaturesAnalysis,
+    )
+    numerical_threshold_config = proto.Field(
+        proto.MESSAGE, number=3, message=ThresholdConfig,
+    )
+    categorical_threshold_config = proto.Field(
+        proto.MESSAGE, number=4, message=ThresholdConfig,
+    )
+
+
+__all__ = tuple(sorted(__protobuf__.manifest))