feat: add pointwise and pairwise metrics to evaluation service

Google APIs · copybara-github · commit 634d59e1f565 · 2024-08-06T09:25:54.000-07:00
PiperOrigin-RevId: 659984672
diff --git a/google/cloud/aiplatform/v1beta1/evaluation_service.proto b/google/cloud/aiplatform/v1beta1/evaluation_service.proto
@@ -121,6 +121,12 @@ message EvaluateInstancesRequest {
     // metric.
     QuestionAnsweringCorrectnessInput question_answering_correctness_input = 18;
 
+    // Input for pointwise metric.
+    PointwiseMetricInput pointwise_metric_input = 28;
+
+    // Input for pairwise metric.
+    PairwiseMetricInput pairwise_metric_input = 29;
+
     // Tool call metric instances.
     // Input for tool call valid metric.
     ToolCallValidInput tool_call_valid_input = 19;
@@ -210,6 +216,13 @@ message EvaluateInstancesResponse {
     QuestionAnsweringCorrectnessResult question_answering_correctness_result =
         17;
 
+    // Generic metrics.
+    // Result for pointwise metric.
+    PointwiseMetricResult pointwise_metric_result = 27;
+
+    // Result for pairwise metric.
+    PairwiseMetricResult pairwise_metric_result = 28;
+
     // Tool call metrics.
     //  Results for tool call valid metric.
     ToolCallValidResults tool_call_valid_results = 18;
@@ -957,6 +970,81 @@ message QuestionAnsweringCorrectnessResult {
   optional float confidence = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
+// Input for pointwise metric.
+message PointwiseMetricInput {
+  // Required. Spec for pointwise metric.
+  PointwiseMetricSpec metric_spec = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Pointwise metric instance.
+  PointwiseMetricInstance instance = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// Pointwise metric instance. Usually one instance corresponds to one row in an
+// evaluation dataset.
+message PointwiseMetricInstance {
+  // Instance for pointwise metric.
+  oneof instance {
+    // Instance specified as a json string. String key-value pairs are expected
+    // in the json_instance to render
+    // PointwiseMetricSpec.instance_prompt_template.
+    string json_instance = 1;
+  }
+}
+
+// Spec for pointwise metric.
+message PointwiseMetricSpec {
+  // Required. Metric prompt template for pointwise metric.
+  optional string metric_prompt_template = 1
+      [(google.api.field_behavior) = REQUIRED];
+}
+
+// Spec for pointwise metric result.
+message PointwiseMetricResult {
+  // Output only. Pointwise metric score.
+  optional float score = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Explanation for pointwise metric score.
+  string explanation = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Input for pairwise metric.
+message PairwiseMetricInput {
+  // Required. Spec for pairwise metric.
+  PairwiseMetricSpec metric_spec = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Pairwise metric instance.
+  PairwiseMetricInstance instance = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// Pairwise metric instance. Usually one instance corresponds to one row in an
+// evaluation dataset.
+message PairwiseMetricInstance {
+  // Instance for pairwise metric.
+  oneof instance {
+    // Instance specified as a json string. String key-value pairs are expected
+    // in the json_instance to render
+    // PairwiseMetricSpec.instance_prompt_template.
+    string json_instance = 1;
+  }
+}
+
+// Spec for pairwise metric.
+message PairwiseMetricSpec {
+  // Required. Metric prompt template for pairwise metric.
+  optional string metric_prompt_template = 1
+      [(google.api.field_behavior) = REQUIRED];
+}
+
+// Spec for pairwise metric result.
+message PairwiseMetricResult {
+  // Output only. Pairwise metric choice.
+  PairwiseChoice pairwise_choice = 1
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Explanation for pairwise metric score.
+  string explanation = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
 // Input for tool call valid metric.
 message ToolCallValidInput {
   // Required. Spec for tool call valid metric.
