|
| 1 | +// Copyright 2022 Google LLC |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +syntax = "proto3"; |
| 16 | + |
| 17 | +package google.cloud.aiplatform.v1; |
| 18 | + |
| 19 | +import "google/api/field_behavior.proto"; |
| 20 | +import "google/cloud/aiplatform/v1/explanation.proto"; |
| 21 | +import "google/protobuf/struct.proto"; |
| 22 | + |
| 23 | +option csharp_namespace = "Google.Cloud.AIPlatform.V1"; |
| 24 | +option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb"; |
| 25 | +option java_multiple_files = true; |
| 26 | +option java_outer_classname = "EvaluatedAnnotationProto"; |
| 27 | +option java_package = "com.google.cloud.aiplatform.v1"; |
| 28 | +option php_namespace = "Google\\Cloud\\AIPlatform\\V1"; |
| 29 | +option ruby_package = "Google::Cloud::AIPlatform::V1"; |
| 30 | + |
| 31 | +// True positive, false positive, or false negative. |
| 32 | +// |
| 33 | +// EvaluatedAnnotation is only available under ModelEvaluationSlice with slice |
| 34 | +// of `annotationSpec` dimension. |
| 35 | +message EvaluatedAnnotation { |
| 36 | + // Describes the type of the EvaluatedAnnotation. The type is determined |
| 37 | + enum EvaluatedAnnotationType { |
| 38 | + // Invalid value. |
| 39 | + EVALUATED_ANNOTATION_TYPE_UNSPECIFIED = 0; |
| 40 | + |
| 41 | + // The EvaluatedAnnotation is a true positive. It has a prediction created |
| 42 | + // by the Model and a ground truth Annotation which the prediction matches. |
| 43 | + TRUE_POSITIVE = 1; |
| 44 | + |
| 45 | + // The EvaluatedAnnotation is false positive. It has a prediction created by |
| 46 | + // the Model which does not match any ground truth annotation. |
| 47 | + FALSE_POSITIVE = 2; |
| 48 | + |
| 49 | + // The EvaluatedAnnotation is false negative. It has a ground truth |
| 50 | + // annotation which is not matched by any of the model created predictions. |
| 51 | + FALSE_NEGATIVE = 3; |
| 52 | + } |
| 53 | + |
| 54 | + // Output only. Type of the EvaluatedAnnotation. |
| 55 | + EvaluatedAnnotationType type = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; |
| 56 | + |
| 57 | + // Output only. The model predicted annotations. |
| 58 | + // |
| 59 | + // For true positive, there is one and only one prediction, which matches the |
| 60 | + // only one ground truth annotation in |
| 61 | + // [ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths]. |
| 62 | + // |
| 63 | + // For false positive, there is one and only one prediction, which doesn't |
| 64 | + // match any ground truth annotation of the corresponding |
| 65 | + // [data_item_view_id][EvaluatedAnnotation.data_item_view_id]. |
| 66 | + // |
| 67 | + // For false negative, there are zero or more predictions which are similar to |
| 68 | + // the only ground truth annotation in |
| 69 | + // [ground_truths][google.cloud.aiplatform.v1.EvaluatedAnnotation.ground_truths] |
| 70 | + // but not enough for a match. |
| 71 | + // |
| 72 | + // The schema of the prediction is stored in |
| 73 | + // [ModelEvaluation.annotation_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.annotation_schema_uri] |
| 74 | + repeated google.protobuf.Value predictions = 2 |
| 75 | + [(google.api.field_behavior) = OUTPUT_ONLY]; |
| 76 | + |
| 77 | + // Output only. The ground truth Annotations, i.e. the Annotations that exist |
| 78 | + // in the test data the Model is evaluated on. |
| 79 | + // |
| 80 | + // For true positive, there is one and only one ground truth annotation, which |
| 81 | + // matches the only prediction in |
| 82 | + // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions]. |
| 83 | + // |
| 84 | + // For false positive, there are zero or more ground truth annotations that |
| 85 | + // are similar to the only prediction in |
| 86 | + // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions], |
| 87 | + // but not enough for a match. |
| 88 | + // |
| 89 | + // For false negative, there is one and only one ground truth annotation, |
| 90 | + // which doesn't match any predictions created by the model. |
| 91 | + // |
| 92 | + // The schema of the ground truth is stored in |
| 93 | + // [ModelEvaluation.annotation_schema_uri][google.cloud.aiplatform.v1.ModelEvaluation.annotation_schema_uri] |
| 94 | + repeated google.protobuf.Value ground_truths = 3 |
| 95 | + [(google.api.field_behavior) = OUTPUT_ONLY]; |
| 96 | + |
| 97 | + // Output only. The data item payload that the Model predicted this |
| 98 | + // EvaluatedAnnotation on. |
| 99 | + google.protobuf.Value data_item_payload = 5 |
| 100 | + [(google.api.field_behavior) = OUTPUT_ONLY]; |
| 101 | + |
| 102 | + // Output only. ID of the EvaluatedDataItemView under the same ancestor |
| 103 | + // ModelEvaluation. The EvaluatedDataItemView consists of all ground truths |
| 104 | + // and predictions on |
| 105 | + // [data_item_payload][google.cloud.aiplatform.v1.EvaluatedAnnotation.data_item_payload]. |
| 106 | + // |
| 107 | + // Can be passed in |
| 108 | + // [GetEvaluatedDataItemView's][ModelService.GetEvaluatedDataItemView][] |
| 109 | + // [id][GetEvaluatedDataItemViewRequest.id]. |
| 110 | + string evaluated_data_item_view_id = 6 |
| 111 | + [(google.api.field_behavior) = OUTPUT_ONLY]; |
| 112 | + |
| 113 | + // Explanations of |
| 114 | + // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions]. |
| 115 | + // Each element of the explanations indicates the explanation for one |
| 116 | + // explanation Method. |
| 117 | + // |
| 118 | + // The attributions list in the |
| 119 | + // [EvaluatedAnnotationExplanation.explanation][google.cloud.aiplatform.v1.EvaluatedAnnotationExplanation.explanation] |
| 120 | + // object corresponds to the |
| 121 | + // [predictions][google.cloud.aiplatform.v1.EvaluatedAnnotation.predictions] |
| 122 | + // list. For example, the second element in the attributions list explains the |
| 123 | + // second element in the predictions list. |
| 124 | + repeated EvaluatedAnnotationExplanation explanations = 8; |
| 125 | + |
| 126 | + // Annotations of model error analysis results. |
| 127 | + repeated ErrorAnalysisAnnotation error_analysis_annotations = 9; |
| 128 | +} |
| 129 | + |
| 130 | +// Explanation result of the prediction produced by the Model. |
| 131 | +message EvaluatedAnnotationExplanation { |
| 132 | + // Explanation type. |
| 133 | + // |
| 134 | + // For AutoML Image Classification models, possible values are: |
| 135 | + // |
| 136 | + // * `image-integrated-gradients` |
| 137 | + // * `image-xrai` |
| 138 | + string explanation_type = 1; |
| 139 | + |
| 140 | + // Explanation attribution response details. |
| 141 | + Explanation explanation = 2; |
| 142 | +} |
| 143 | + |
| 144 | +// Model error analysis for each annotation. |
| 145 | +message ErrorAnalysisAnnotation { |
| 146 | + // Attributed items for a given annotation, typically representing neighbors |
| 147 | + // from the training sets constrained by the query type. |
| 148 | + message AttributedItem { |
| 149 | + // The unique ID for each annotation. Used by FE to allocate the annotation |
| 150 | + // in DB. |
| 151 | + string annotation_resource_name = 1; |
| 152 | + |
| 153 | + // The distance of this item to the annotation. |
| 154 | + double distance = 2; |
| 155 | + } |
| 156 | + |
| 157 | + // The query type used for finding the attributed items. |
| 158 | + enum QueryType { |
| 159 | + // Unspecified query type for model error analysis. |
| 160 | + QUERY_TYPE_UNSPECIFIED = 0; |
| 161 | + |
| 162 | + // Query similar samples across all classes in the dataset. |
| 163 | + ALL_SIMILAR = 1; |
| 164 | + |
| 165 | + // Query similar samples from the same class of the input sample. |
| 166 | + SAME_CLASS_SIMILAR = 2; |
| 167 | + |
| 168 | + // Query dissimilar samples from the same class of the input sample. |
| 169 | + SAME_CLASS_DISSIMILAR = 3; |
| 170 | + } |
| 171 | + |
| 172 | + // Attributed items for a given annotation, typically representing neighbors |
| 173 | + // from the training sets constrained by the query type. |
| 174 | + repeated AttributedItem attributed_items = 1; |
| 175 | + |
| 176 | + // The query type used for finding the attributed items. |
| 177 | + QueryType query_type = 2; |
| 178 | + |
| 179 | + // The outlier score of this annotated item. Usually defined as the min of all |
| 180 | + // distances from attributed items. |
| 181 | + double outlier_score = 3; |
| 182 | + |
| 183 | + // The threshold used to determine if this annotation is an outlier or not. |
| 184 | + double outlier_threshold = 4; |
| 185 | +} |
0 commit comments