Skip to content

Commit 9b94dba

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: Add Feature Monitoring API to Feature Store
docs: update feature creation message commentary PiperOrigin-RevId: 692336095
1 parent d54f4e9 commit 9b94dba

6 files changed

Lines changed: 618 additions & 9 deletions

File tree

‎google/cloud/aiplatform/v1beta1/BUILD.bazel‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ proto_library(
5757
"extension_registry_service.proto",
5858
"feature.proto",
5959
"feature_group.proto",
60+
"feature_monitor.proto",
61+
"feature_monitor_job.proto",
6062
"feature_monitoring_stats.proto",
6163
"feature_online_store.proto",
6264
"feature_online_store_admin_service.proto",
@@ -120,10 +122,10 @@ proto_library(
120122
"pipeline_state.proto",
121123
"prediction_service.proto",
122124
"publisher_model.proto",
123-
"reservation_affinity.proto",
124125
"reasoning_engine.proto",
125126
"reasoning_engine_execution_service.proto",
126127
"reasoning_engine_service.proto",
128+
"reservation_affinity.proto",
127129
"saved_query.proto",
128130
"schedule.proto",
129131
"schedule_service.proto",

‎google/cloud/aiplatform/v1beta1/feature.proto‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package google.cloud.aiplatform.v1beta1;
1818

1919
import "google/api/field_behavior.proto";
2020
import "google/api/resource.proto";
21+
import "google/cloud/aiplatform/v1beta1/feature_monitor.proto";
2122
import "google/cloud/aiplatform/v1beta1/feature_monitoring_stats.proto";
2223
import "google/cloud/aiplatform/v1beta1/featurestore_monitoring.proto";
2324
import "google/protobuf/timestamp.proto";
@@ -196,6 +197,11 @@ message Feature {
196197
repeated MonitoringStatsAnomaly monitoring_stats_anomalies = 11
197198
[(google.api.field_behavior) = OUTPUT_ONLY];
198199

200+
// Output only. Only applicable for Vertex AI Feature Store.
201+
// The list of historical stats and anomalies.
202+
repeated FeatureStatsAndAnomaly feature_stats_and_anomaly = 13
203+
[(google.api.field_behavior) = OUTPUT_ONLY];
204+
199205
// Only applicable for Vertex AI Feature Store.
200206
// The name of the BigQuery Table/View column hosting data for this version.
201207
// If no value is provided, will use feature_id.

‎google/cloud/aiplatform/v1beta1/feature_monitor.proto‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.aiplatform.v1beta1;
18+
19+
import "google/api/field_behavior.proto";
20+
import "google/api/resource.proto";
21+
import "google/protobuf/struct.proto";
22+
import "google/protobuf/timestamp.proto";
23+
import "google/type/interval.proto";
24+
25+
option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
26+
option go_package = "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb;aiplatformpb";
27+
option java_multiple_files = true;
28+
option java_outer_classname = "FeatureMonitorProto";
29+
option java_package = "com.google.cloud.aiplatform.v1beta1";
30+
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
31+
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";
32+
33+
// Vertex AI Feature Monitor.
34+
message FeatureMonitor {
35+
option (google.api.resource) = {
36+
type: "aiplatform.googleapis.com/FeatureMonitor"
37+
pattern: "projects/{project}/locations/{location}/featureGroups/{feature_group}/featureMonitors/{feature_monitor}"
38+
plural: "featureMonitors"
39+
singular: "featureMonitor"
40+
};
41+
42+
// Identifier. Name of the FeatureMonitor. Format:
43+
// `projects/{project}/locations/{location}/featureGroups/{featureGroup}/featureMonitors/{featureMonitor}`
44+
string name = 1 [(google.api.field_behavior) = IDENTIFIER];
45+
46+
// Output only. Timestamp when this FeatureMonitor was created.
47+
google.protobuf.Timestamp create_time = 2
48+
[(google.api.field_behavior) = OUTPUT_ONLY];
49+
50+
// Output only. Timestamp when this FeatureMonitor was last updated.
51+
google.protobuf.Timestamp update_time = 3
52+
[(google.api.field_behavior) = OUTPUT_ONLY];
53+
54+
// Optional. Used to perform consistent read-modify-write updates. If not set,
55+
// a blind "overwrite" update happens.
56+
string etag = 4 [(google.api.field_behavior) = OPTIONAL];
57+
58+
// Optional. The labels with user-defined metadata to organize your
59+
// FeatureMonitor.
60+
//
61+
// Label keys and values can be no longer than 64 characters
62+
// (Unicode codepoints), can only contain lowercase letters, numeric
63+
// characters, underscores and dashes. International characters are allowed.
64+
//
65+
// See https://goo.gl/xmQnxf for more information on and examples of labels.
66+
// No more than 64 user labels can be associated with one
67+
// FeatureMonitor(System labels are excluded)." System reserved label keys
68+
// are prefixed with "aiplatform.googleapis.com/" and are immutable.
69+
map<string, string> labels = 5 [(google.api.field_behavior) = OPTIONAL];
70+
71+
// Optional. Description of the FeatureMonitor.
72+
string description = 6 [(google.api.field_behavior) = OPTIONAL];
73+
74+
// Required. Schedule config for the FeatureMonitor.
75+
ScheduleConfig schedule_config = 7 [(google.api.field_behavior) = REQUIRED];
76+
77+
// Required. Feature selection config for the FeatureMonitor.
78+
FeatureSelectionConfig feature_selection_config = 8
79+
[(google.api.field_behavior) = REQUIRED];
80+
}
81+
82+
// Schedule configuration for the FeatureMonitor.
83+
message ScheduleConfig {
84+
// Cron schedule (https://en.wikipedia.org/wiki/Cron) to launch scheduled
85+
// runs. To explicitly set a timezone to the cron tab, apply a prefix in
86+
// the cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or "TZ=${IANA_TIME_ZONE}".
87+
// The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone
88+
// database. For example, "CRON_TZ=America/New_York 1 * * * *", or
89+
// "TZ=America/New_York 1 * * * *".
90+
string cron = 1;
91+
}
92+
93+
// Feature selection configuration for the FeatureMonitor.
94+
message FeatureSelectionConfig {
95+
// Feature configuration.
96+
message FeatureConfig {
97+
// Required. The ID of the feature resource. Final component of the
98+
// Feature's resource name.
99+
string feature_id = 1 [(google.api.field_behavior) = REQUIRED];
100+
101+
// Optional. Drift threshold. If calculated difference with baseline data
102+
// larger than threshold, it will be considered as the feature has drift. If
103+
// not present, the threshold will be default to 0.3.
104+
double drift_threshold = 2 [(google.api.field_behavior) = OPTIONAL];
105+
}
106+
107+
// Optional. A list of features to be monitored and each feature's drift
108+
// threshold.
109+
repeated FeatureConfig feature_configs = 1
110+
[(google.api.field_behavior) = OPTIONAL];
111+
}
112+
113+
// Stats and Anomaly generated by FeatureMonitorJobs. Anomaly only includes
114+
// Drift.
115+
message FeatureStatsAndAnomaly {
116+
// Feature Id.
117+
string feature_id = 1;
118+
119+
// Feature stats. e.g. histogram buckets.
120+
// In the format of tensorflow.metadata.v0.DatasetFeatureStatistics.
121+
google.protobuf.Value feature_stats = 2;
122+
123+
// Deviation from the current stats to baseline stats.
124+
// 1. For categorical feature, the distribution distance is calculated by
125+
// L-inifinity norm.
126+
// 2. For numerical feature, the distribution distance is calculated by
127+
// Jensen–Shannon divergence.
128+
double distribution_deviation = 3;
129+
130+
// This is the threshold used when detecting drifts, which is set in
131+
// FeatureMonitor.FeatureSelectionConfig.FeatureConfig.drift_threshold
132+
double drift_detection_threshold = 4;
133+
134+
// If set to true, indicates current stats is detected as and comparing
135+
// with baseline stats.
136+
bool drift_detected = 5;
137+
138+
// The timestamp we take snapshot for feature values to generate stats.
139+
google.protobuf.Timestamp stats_time = 6;
140+
141+
// The ID of the FeatureMonitorJob that generated this FeatureStatsAndAnomaly.
142+
int64 feature_monitor_job_id = 7;
143+
144+
// The ID of the FeatureMonitor that this FeatureStatsAndAnomaly generated
145+
// according to.
146+
string feature_monitor_id = 8;
147+
}
148+
149+
// Defines how to select FeatureStatsAndAnomaly to be populated in response.
150+
// If set, retrieves FeatureStatsAndAnomaly generated by FeatureMonitors based
151+
// on this spec.
152+
message FeatureStatsAndAnomalySpec {
153+
// Optional. If set, returns the most recent count of stats. Valid value is
154+
// [0, 100]. If stats_time_range is set, return most recent count of stats
155+
// within the stats_time_range.
156+
optional int32 latest_stats_count = 1
157+
[(google.api.field_behavior) = OPTIONAL];
158+
159+
// Optional. If set, return all stats generated between [start_time,
160+
// end_time). If latest_stats_count is set, return the most recent count of
161+
// stats within the stats_time_range.
162+
google.type.Interval stats_time_range = 2
163+
[(google.api.field_behavior) = OPTIONAL];
164+
}

‎google/cloud/aiplatform/v1beta1/feature_monitor_job.proto‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.aiplatform.v1beta1;
18+
19+
import "google/api/field_behavior.proto";
20+
import "google/api/resource.proto";
21+
import "google/cloud/aiplatform/v1beta1/feature_monitor.proto";
22+
import "google/protobuf/timestamp.proto";
23+
import "google/rpc/status.proto";
24+
25+
option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
26+
option go_package = "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb;aiplatformpb";
27+
option java_multiple_files = true;
28+
option java_outer_classname = "FeatureMonitorJobProto";
29+
option java_package = "com.google.cloud.aiplatform.v1beta1";
30+
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
31+
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";
32+
33+
// Vertex AI Feature Monitor Job.
34+
message FeatureMonitorJob {
35+
option (google.api.resource) = {
36+
type: "aiplatform.googleapis.com/FeatureMonitorJob"
37+
pattern: "projects/{project}/locations/{location}/featureGroups/{feature_group}/featureMonitors/{feature_monitor}/featureMonitorJobs/{feature_monitor_job}"
38+
plural: "featureMonitorJobs"
39+
singular: "featureMonitorJob"
40+
};
41+
42+
// Summary from the FeatureMonitorJob.
43+
message JobSummary {
44+
// Output only. BigQuery slot milliseconds consumed.
45+
int64 total_slot_ms = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
46+
47+
// Output only. Features and their stats and anomalies
48+
repeated FeatureStatsAndAnomaly feature_stats_and_anomalies = 2
49+
[(google.api.field_behavior) = OUTPUT_ONLY];
50+
}
51+
52+
// Choices of the trigger type.
53+
enum FeatureMonitorJobTrigger {
54+
// Trigger type unspecified.
55+
FEATURE_MONITOR_JOB_TRIGGER_UNSPECIFIED = 0;
56+
57+
// Triggered by periodic schedule.
58+
FEATURE_MONITOR_JOB_TRIGGER_PERIODIC = 1;
59+
60+
// Triggered on demand by CreateFeatureMonitorJob request.
61+
FEATURE_MONITOR_JOB_TRIGGER_ON_DEMAND = 2;
62+
}
63+
64+
// Identifier. Name of the FeatureMonitorJob. Format:
65+
// `projects/{project}/locations/{location}/featureGroups/{feature_group}/featureMonitors/{feature_monitor}/featureMonitorJobs/{feature_monitor_job}`.
66+
string name = 1 [(google.api.field_behavior) = IDENTIFIER];
67+
68+
// Output only. Timestamp when this FeatureMonitorJob was created. Creation of
69+
// a FeatureMonitorJob means that the job is pending / waiting for sufficient
70+
// resources but may not have started running yet.
71+
google.protobuf.Timestamp create_time = 2
72+
[(google.api.field_behavior) = OUTPUT_ONLY];
73+
74+
// Output only. Final status of the FeatureMonitorJob.
75+
google.rpc.Status final_status = 3
76+
[(google.api.field_behavior) = OUTPUT_ONLY];
77+
78+
// Output only. Summary from the FeatureMonitorJob.
79+
JobSummary job_summary = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
80+
81+
// Optional. The labels with user-defined metadata to organize your
82+
// FeatureMonitorJob.
83+
//
84+
// Label keys and values can be no longer than 64 characters
85+
// (Unicode codepoints), can only contain lowercase letters, numeric
86+
// characters, underscores and dashes. International characters are allowed.
87+
//
88+
// See https://goo.gl/xmQnxf for more information on and examples of labels.
89+
// No more than 64 user labels can be associated with one
90+
// FeatureMonitor(System labels are excluded)." System reserved label keys
91+
// are prefixed with "aiplatform.googleapis.com/" and are immutable.
92+
map<string, string> labels = 5 [(google.api.field_behavior) = OPTIONAL];
93+
94+
// Optional. Description of the FeatureMonitor.
95+
string description = 6 [(google.api.field_behavior) = OPTIONAL];
96+
97+
// Output only. FeatureMonitorJob ID comparing to which the drift is
98+
// calculated.
99+
int64 drift_base_feature_monitor_job_id = 7
100+
[(google.api.field_behavior) = OUTPUT_ONLY];
101+
102+
// Output only. Data snapshot time comparing to which the drift is calculated.
103+
google.protobuf.Timestamp drift_base_snapshot_time = 8
104+
[(google.api.field_behavior) = OUTPUT_ONLY];
105+
106+
// Output only. Feature selection config used when creating FeatureMonitorJob.
107+
FeatureSelectionConfig feature_selection_config = 9
108+
[(google.api.field_behavior) = OUTPUT_ONLY];
109+
110+
// Output only. Trigger type of the Feature Monitor Job.
111+
FeatureMonitorJobTrigger trigger_type = 10
112+
[(google.api.field_behavior) = OUTPUT_ONLY];
113+
}

0 commit comments

Comments
 (0)