Skip to content

Commit 3290633

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: [generativelanguage] add support for additional reasons to stop image generation (#6820)
* feat: add support for FunctionResponsePart feat: add support for raw media bytes for function response feat: add Computer Use tool type feat: add support for image generation features feat: add support for additional reasons to stop image generation feat: add URL_RETRIEVAL_STATUS_UNSAFE and URL_RETRIEVAL_STATUS_PAYWALL statuses for url retrieval feat: add log_probability_sum fix!: replaced GenerateVideoResponse by PredictLongRunningGeneratedVideoResponse field PiperOrigin-RevId: 819821787 Source-Link: googleapis/googleapis@fd84be8 Source-Link: googleapis/googleapis-gen@eaa1254 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiZWFhMTI1NDc5Nzc1YzI3YzFhMGE2NzgxODQzY2NiZDc1MTAyMGE2NSJ9 * feat: add support for additional reasons to stop image generation feat: add support for finish_message that contain details the reason why the model stopped generating tokens feat: add URL_RETRIEVAL_STATUS_UNSAFE and URL_RETRIEVAL_STATUS_PAYWALL statuses for url retrieval feat: add log_probability_sum docs: deprecation notice for HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY PiperOrigin-RevId: 819823490 Source-Link: googleapis/googleapis@f495b45 Source-Link: googleapis/googleapis-gen@b8008e9 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiYjgwMDhlOTQyN2ExNjg3ZDk5MzJhZjgwNGFlOTA1ZDU1NmRhNDIyNSJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 68fd63c commit 3290633

12 files changed

Lines changed: 2454 additions & 147 deletions

File tree

‎packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1/generative_service.proto‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import "google/api/annotations.proto";
2323
import "google/api/client.proto";
2424
import "google/api/field_behavior.proto";
2525
import "google/api/resource.proto";
26+
import "google/protobuf/struct.proto";
2627

2728
option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1/generativelanguagepb;generativelanguagepb";
2829
option java_multiple_files = true;
@@ -144,6 +145,7 @@ enum TaskType {
144145
}
145146

146147
// Request to generate a completion from the model.
148+
// NEXT ID: 18
147149
message GenerateContentRequest {
148150
// Required. The name of the `Model` to use for generating the completion.
149151
//
@@ -190,6 +192,7 @@ message GenerateContentRequest {
190192

191193
// Configuration options for model generation and outputs. Not all parameters
192194
// are configurable for every model.
195+
// Next ID: 29
193196
message GenerationConfig {
194197
// Optional. Number of generated responses to return. If unset, this will
195198
// default to 1. Please note that this doesn't work for previous generation
@@ -248,6 +251,11 @@ message GenerationConfig {
248251
// generated seed.
249252
optional int32 seed = 8 [(google.api.field_behavior) = OPTIONAL];
250253

254+
// Optional. An internal detail. Use `responseJsonSchema` rather than this
255+
// field.
256+
google.protobuf.Value response_json_schema_ordered = 28
257+
[(google.api.field_behavior) = OPTIONAL];
258+
251259
// Optional. Presence penalty applied to the next token's logprobs if the
252260
// token has already been seen in the response.
253261
//
@@ -289,6 +297,7 @@ message GenerationConfig {
289297
// [response_logprobs=True][google.ai.generativelanguage.v1.GenerationConfig.response_logprobs].
290298
// This sets the number of top logprobs to return at each decoding step in the
291299
// [Candidate.logprobs_result][google.ai.generativelanguage.v1.Candidate.logprobs_result].
300+
// The number must be in the range of [0, 20].
292301
optional int32 logprobs = 18 [(google.api.field_behavior) = OPTIONAL];
293302

294303
// Optional. Enables enhanced civic answers. It may not be available for all
@@ -442,8 +451,22 @@ message Candidate {
442451
// violations.
443452
IMAGE_SAFETY = 11;
444453

454+
// Image generation stopped because generated images has other prohibited
455+
// content.
456+
IMAGE_PROHIBITED_CONTENT = 14;
457+
458+
// Image generation stopped because of other miscellaneous issue.
459+
IMAGE_OTHER = 15;
460+
461+
// The model was expected to generate an image, but none was generated.
462+
NO_IMAGE = 16;
463+
445464
// Model generated a tool call but no tools were enabled in the request.
446465
UNEXPECTED_TOOL_CALL = 12;
466+
467+
// Model called too many tools consecutively, thus the system exited
468+
// execution.
469+
TOO_MANY_TOOL_CALLS = 13;
447470
}
448471

449472
// Output only. Index of the candidate in the list of response candidates.
@@ -460,6 +483,13 @@ message Candidate {
460483
(google.api.field_behavior) = OUTPUT_ONLY
461484
];
462485

486+
// Optional. Output only. Details the reason why the model stopped generating
487+
// tokens. This is populated only when `finish_reason` is set.
488+
optional string finish_message = 4 [
489+
(google.api.field_behavior) = OPTIONAL,
490+
(google.api.field_behavior) = OUTPUT_ONLY
491+
];
492+
463493
// List of ratings for the safety of a response candidate.
464494
//
465495
// There is at most one rating per category.
@@ -512,6 +542,12 @@ message UrlMetadata {
512542

513543
// Url retrieval is failed due to error.
514544
URL_RETRIEVAL_STATUS_ERROR = 2;
545+
546+
// Url retrieval is failed because the content is behind paywall.
547+
URL_RETRIEVAL_STATUS_PAYWALL = 3;
548+
549+
// Url retrieval is failed because the content is unsafe.
550+
URL_RETRIEVAL_STATUS_UNSAFE = 4;
515551
}
516552

517553
// Retrieved url by the tool.
@@ -541,6 +577,9 @@ message LogprobsResult {
541577
repeated Candidate candidates = 1;
542578
}
543579

580+
// Sum of log probabilities for all tokens.
581+
optional float log_probability_sum = 3;
582+
544583
// Length = total number of decoding steps.
545584
repeated TopCandidates top_candidates = 1;
546585

‎packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1/safety.proto‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ enum HarmCategory {
6565
HARM_CATEGORY_DANGEROUS_CONTENT = 10;
6666

6767
// **Gemini** - Content that may be used to harm civic integrity.
68-
HARM_CATEGORY_CIVIC_INTEGRITY = 11;
68+
// DEPRECATED: use enable_enhanced_civic_answers instead.
69+
HARM_CATEGORY_CIVIC_INTEGRITY = 11 [deprecated = true];
6970
}
7071

7172
// Safety rating for a piece of content.

‎packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/content.proto‎

Lines changed: 92 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,29 @@ message Part {
141141
// Optional. An opaque signature for the thought so it can be reused in
142142
// subsequent requests.
143143
bytes thought_signature = 13 [(google.api.field_behavior) = OPTIONAL];
144+
145+
// Custom metadata associated with the Part.
146+
// Agents using genai.Part as content representation may need to keep track
147+
// of the additional information. For example it can be name of a file/source
148+
// from which the Part originates or a way to multiplex multiple Part streams.
149+
google.protobuf.Struct part_metadata = 8;
150+
}
151+
152+
// A datatype containing media that is part of a `FunctionResponse` message.
153+
//
154+
// A `FunctionResponsePart` consists of data which has an associated datatype. A
155+
// `FunctionResponsePart` can only contain one of the accepted types in
156+
// `FunctionResponsePart.data`.
157+
//
158+
// A `FunctionResponsePart` must have a fixed IANA MIME type identifying the
159+
// type and subtype of the media if the `inline_data` field is filled with raw
160+
// bytes.
161+
message FunctionResponsePart {
162+
// The data of the function response part.
163+
oneof data {
164+
// Inline media bytes.
165+
FunctionResponseBlob inline_data = 1;
166+
}
144167
}
145168

146169
// Raw media bytes.
@@ -160,6 +183,24 @@ message Blob {
160183
bytes data = 2;
161184
}
162185

186+
// Raw media bytes for function response.
187+
//
188+
// Text should not be sent as raw bytes, use the 'FunctionResponse.response'
189+
// field.
190+
message FunctionResponseBlob {
191+
// The IANA standard MIME type of the source data.
192+
// Examples:
193+
// - image/png
194+
// - image/jpeg
195+
// If an unsupported MIME type is provided, an error will be returned. For a
196+
// complete list of supported types, see [Supported file
197+
// formats](https://ai.google.dev/gemini-api/docs/prompting_with_media#supported_file_formats).
198+
string mime_type = 1;
199+
200+
// Raw bytes for media formats.
201+
bytes data = 2;
202+
}
203+
163204
// URI based data.
164205
message FileData {
165206
// Optional. The IANA standard MIME type of the source data.
@@ -242,6 +283,8 @@ message CodeExecutionResult {
242283
// A `Tool` is a piece of code that enables the system to interact with
243284
// external systems to perform an action, or set of actions, outside of
244285
// knowledge and scope of the model.
286+
//
287+
// Next ID: 12
245288
message Tool {
246289
// GoogleSearch tool type.
247290
// Tool to support Google Search in Model. Powered by Google.
@@ -253,6 +296,29 @@ message Tool {
253296
[(google.api.field_behavior) = OPTIONAL];
254297
}
255298

299+
// Computer Use tool type.
300+
message ComputerUse {
301+
// Represents the environment being operated, such as a web browser.
302+
enum Environment {
303+
// Defaults to browser.
304+
ENVIRONMENT_UNSPECIFIED = 0;
305+
306+
// Operates in a web browser.
307+
ENVIRONMENT_BROWSER = 1;
308+
}
309+
310+
// Required. The environment being operated.
311+
Environment environment = 3 [(google.api.field_behavior) = REQUIRED];
312+
313+
// Optional. By default, predefined functions are included in the final
314+
// model call. Some of them can be explicitly excluded from being
315+
// automatically included. This can serve two purposes:
316+
// 1. Using a more restricted / different action space.
317+
// 2. Improving the definitions / instructions of predefined functions.
318+
repeated string excluded_predefined_functions = 5
319+
[(google.api.field_behavior) = OPTIONAL];
320+
}
321+
256322
// Optional. A list of `FunctionDeclarations` available to the model that can
257323
// be used for function calling.
258324
//
@@ -280,6 +346,11 @@ message Tool {
280346
// Tool to support Google Search in Model. Powered by Google.
281347
GoogleSearch google_search = 4 [(google.api.field_behavior) = OPTIONAL];
282348

349+
// Optional. Tool to support the model interacting directly with the computer.
350+
// If enabled, it automatically populates computer-use specific Function
351+
// Declarations.
352+
ComputerUse computer_use = 6 [(google.api.field_behavior) = OPTIONAL];
353+
283354
// Optional. Tool to support URL context retrieval.
284355
UrlContext url_context = 8 [(google.api.field_behavior) = OPTIONAL];
285356
}
@@ -352,6 +423,9 @@ message FunctionCallingConfig {
352423
// Model decides to predict either a function call
353424
// or a natural language response, but will validate function calls with
354425
// constrained decoding.
426+
// If "allowed_function_names" are set, the predicted function call will be
427+
// limited to any one of "allowed_function_names", else the predicted
428+
// function call will be any one of the provided "function_declarations".
355429
VALIDATED = 4;
356430
}
357431

@@ -362,9 +436,9 @@ message FunctionCallingConfig {
362436
// Optional. A set of function names that, when provided, limits the functions
363437
// the model will call.
364438
//
365-
// This should only be set when the Mode is ANY. Function names
366-
// should match [FunctionDeclaration.name]. With mode set to ANY, model will
367-
// predict a function call from the set of function names provided.
439+
// This should only be set when the Mode is ANY or VALIDATED. Function names
440+
// should match [FunctionDeclaration.name]. When set, model will
441+
// predict a function call from only allowed function names.
368442
repeated string allowed_function_names = 2
369443
[(google.api.field_behavior) = OPTIONAL];
370444
}
@@ -392,8 +466,8 @@ message FunctionDeclaration {
392466
}
393467

394468
// Required. The name of the function.
395-
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
396-
// length of 63.
469+
// Must be a-z, A-Z, 0-9, or contain underscores, colons, dots, and dashes,
470+
// with a maximum length of 64.
397471
string name = 1 [(google.api.field_behavior) = REQUIRED];
398472

399473
// Required. A brief description of the function.
@@ -453,7 +527,7 @@ message FunctionCall {
453527

454528
// Required. The name of the function to call.
455529
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
456-
// length of 63.
530+
// length of 64.
457531
string name = 1 [(google.api.field_behavior) = REQUIRED];
458532

459533
// Optional. The function parameters and values in JSON object format.
@@ -491,12 +565,21 @@ message FunctionResponse {
491565

492566
// Required. The name of the function to call.
493567
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
494-
// length of 63.
568+
// length of 64.
495569
string name = 1 [(google.api.field_behavior) = REQUIRED];
496570

497571
// Required. The function response in JSON object format.
572+
// Callers can use any keys of their choice that fit the function's syntax
573+
// to return the function output, e.g. "output", "result", etc.
574+
// In particular, if the function call failed to execute, the response can
575+
// have an "error" key to return error details to the model.
498576
google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
499577

578+
// Optional. Ordered `Parts` that constitute a function response. Parts may
579+
// have different IANA MIME types.
580+
repeated FunctionResponsePart parts = 8
581+
[(google.api.field_behavior) = OPTIONAL];
582+
500583
// Optional. Signals that function call continues, and more responses will be
501584
// returned, turning the function call into a generator.
502585
// Is only applicable to NON_BLOCKING function calls, is ignored otherwise.
@@ -521,11 +604,8 @@ message Schema {
521604
// Required. Data type.
522605
Type type = 1 [(google.api.field_behavior) = REQUIRED];
523606

524-
// Optional. The format of the data. This is used only for primitive
525-
// datatypes. Supported formats:
526-
// for NUMBER type: float, double
527-
// for INTEGER type: int32, int64
528-
// for STRING type: enum, date-time
607+
// Optional. The format of the data. Any value is allowed, but most do not
608+
// trigger any special functionality.
529609
string format = 2 [(google.api.field_behavior) = OPTIONAL];
530610

531611
// Optional. The title of the schema.

‎packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/file.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ message File {
6161

6262
// Indicates the file is generated by Google.
6363
GENERATED = 2;
64+
65+
// Indicates the file is a registered, i.e. a Google Cloud Storage file.
66+
REGISTERED = 3;
6467
}
6568

6669
// Metadata for the File.

0 commit comments

Comments
 (0)