feat: add FunctionResponsePart and excluded_predefined_functions in ComputerUse

Google APIs · copybara-github · commit c288189b43c0 · 2025-10-17T19:42:08.000-07:00
PiperOrigin-RevId: 820916199
diff --git a/google/cloud/aiplatform/v1beta1/tool.proto b/google/cloud/aiplatform/v1beta1/tool.proto
@@ -99,6 +99,15 @@ message Tool {
 
     // Required. The environment being operated.
     Environment environment = 1 [(google.api.field_behavior) = REQUIRED];
+
+    // Optional. By default, [predefined
+    // functions](https://cloud.google.com/vertex-ai/generative-ai/docs/computer-use#supported-actions)
+    // are included in the final model call. Some of them can be explicitly
+    // excluded from being automatically included. This can serve two purposes:
+    // 1. Using a more restricted / different action space.
+    // 2. Improving the definitions / instructions of predefined functions.
+    repeated string excluded_predefined_functions = 2
+        [(google.api.field_behavior) = OPTIONAL];
   }
 
   // Optional. Function tool type.
@@ -275,6 +284,64 @@ message FunctionCall {
   google.protobuf.Struct args = 2 [(google.api.field_behavior) = OPTIONAL];
 }
 
+// A datatype containing media that is part of a `FunctionResponse` message.
+//
+// A `FunctionResponsePart` consists of data which has an associated datatype. A
+// `FunctionResponsePart` can only contain one of the accepted types in
+// `FunctionResponsePart.data`.
+//
+// A `FunctionResponsePart` must have a fixed IANA MIME type identifying the
+// type and subtype of the media if the `inline_data` field is filled with raw
+// bytes.
+message FunctionResponsePart {
+  // The data of the function response part.
+  oneof data {
+    // Inline media bytes.
+    FunctionResponseBlob inline_data = 1;
+
+    // URI based data.
+    FunctionResponseFileData file_data = 2;
+  }
+}
+
+// Raw media bytes for function response.
+//
+// Text should not be sent as raw bytes, use the 'text' field.
+message FunctionResponseBlob {
+  // Required. The IANA standard MIME type of the source data.
+  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Raw bytes.
+  bytes data = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Display name of the blob.
+  //
+  // Used to provide a label or filename to distinguish blobs.
+  //
+  // This field is only returned in PromptMessage for prompt management.
+  // It is currently used in the Gemini GenerateContent calls only when server
+  // side tools (code_execution, google_search, and url_context) are enabled.
+  string display_name = 4 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// URI based data for function response.
+message FunctionResponseFileData {
+  // Required. The IANA standard MIME type of the source data.
+  string mime_type = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. URI.
+  string file_uri = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Display name of the file data.
+  //
+  // Used to provide a label or filename to distinguish file datas.
+  //
+  // This field is only returned in PromptMessage for prompt management.
+  // It is currently used in the Gemini GenerateContent calls only when server
+  // side tools (code_execution, google_search, and url_context) are enabled.
+  string display_name = 3 [(google.api.field_behavior) = OPTIONAL];
+}
+
 // The result output from a [FunctionCall] that contains a string representing
 // the [FunctionDeclaration.name] and a structured JSON object containing any
 // output from the function is used as context to the model. This should contain
@@ -293,6 +360,11 @@ message FunctionResponse {
   // error details (if any). If "output" and "error" keys are not specified,
   // then whole "response" is treated as function output.
   google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Ordered `Parts` that constitute a function response. Parts may
+  // have different IANA MIME types.
+  repeated FunctionResponsePart parts = 4
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Code generated by the model that is meant to be executed, and the result
