Skip to content

Commit 793b65c

Browse files
shethaaditAdit Shethmarkwallace-microsoftrogerbarreto
authored
.Net: Add AudioTimestamp Property to GeminiPromptExecutionSettings (#9526)
### Motivation and Context **Why is this change required?** This change is required to enhance the functionality of audio processing in the Semantic Kernel by allowing the inclusion of timestamps in audio responses. **What problem does it solve?** It resolves the limitation of missing audio timestamp support in the `GeminiPromptExecutionSettings`, which has been a significant blocker for developers trying to utilize audio prompts effectively. **What scenario does it contribute to?** This contribution is particularly useful for applications that require precise audio response management, enabling better synchronization and analysis of audio data in real-time scenarios. If it fixes an open issue, please link to the issue here. #9500 ### Description This contribution adds a new feature that allows audio responses to include timestamps, a crucial element for many applications. Here’s a quick overview of what’s been implemented: **New Property:** The `AudioTimestamp` property indicates whether the audio response should include timestamps. When enabled, this property will ensure that audio timestamps are included in requests to the model, providing developers with the precise control they need for their applications. **Seamless Integration:** I conducted a thorough review of the existing codebase to ensure that this new feature integrates smoothly without disrupting current functionalities. This change is designed to be backward-compatible, making it easier for developers to adopt. **Future-Proofing:** By adding this property, we’re not just solving an immediate need; we’re also paving the way for future enhancements in audio processing capabilities within the Semantic Kernel framework. ### Contribution Checklist - [Y] The code builds clean without any errors or warnings - [Y] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [Y] All unit tests pass, and I have added new tests where possible - [Y] I didn't break anyone 😄 --------- Co-authored-by: Adit Sheth <[email protected]> Co-authored-by: Mark Wallace <[email protected]> Co-authored-by: Roger Barreto <[email protected]>
1 parent 803fb33 commit 793b65c

5 files changed

Lines changed: 41 additions & 3 deletions

File tree

‎dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/Clients/GeminiChatGenerationTests.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,8 @@ public async Task ShouldUsePromptExecutionSettingsAsync()
236236
{
237237
MaxTokens = 102,
238238
Temperature = 0.45,
239-
TopP = 0.6
239+
TopP = 0.6,
240+
AudioTimestamp = true
240241
};
241242

242243
// Act
@@ -247,6 +248,7 @@ public async Task ShouldUsePromptExecutionSettingsAsync()
247248
Assert.NotNull(geminiRequest);
248249
Assert.Equal(executionSettings.MaxTokens, geminiRequest.Configuration!.MaxOutputTokens);
249250
Assert.Equal(executionSettings.Temperature, geminiRequest.Configuration!.Temperature);
251+
Assert.Equal(executionSettings.AudioTimestamp, geminiRequest.Configuration!.AudioTimestamp);
250252
Assert.Equal(executionSettings.TopP, geminiRequest.Configuration!.TopP);
251253
}
252254

‎dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public void FromPromptItReturnsWithConfiguration()
2424
Temperature = 1.5,
2525
MaxTokens = 10,
2626
TopP = 0.9,
27+
AudioTimestamp = true
2728
};
2829

2930
// Act
@@ -33,6 +34,7 @@ public void FromPromptItReturnsWithConfiguration()
3334
Assert.NotNull(request.Configuration);
3435
Assert.Equal(executionSettings.Temperature, request.Configuration.Temperature);
3536
Assert.Equal(executionSettings.MaxTokens, request.Configuration.MaxOutputTokens);
37+
Assert.Equal(executionSettings.AudioTimestamp, request.Configuration.AudioTimestamp);
3638
Assert.Equal(executionSettings.TopP, request.Configuration.TopP);
3739
}
3840

@@ -85,6 +87,7 @@ public void FromChatHistoryItReturnsWithConfiguration()
8587
Temperature = 1.5,
8688
MaxTokens = 10,
8789
TopP = 0.9,
90+
AudioTimestamp = true
8891
};
8992

9093
// Act
@@ -94,6 +97,7 @@ public void FromChatHistoryItReturnsWithConfiguration()
9497
Assert.NotNull(request.Configuration);
9598
Assert.Equal(executionSettings.Temperature, request.Configuration.Temperature);
9699
Assert.Equal(executionSettings.MaxTokens, request.Configuration.MaxOutputTokens);
100+
Assert.Equal(executionSettings.AudioTimestamp, request.Configuration.AudioTimestamp);
97101
Assert.Equal(executionSettings.TopP, request.Configuration.TopP);
98102
}
99103

‎dotnet/src/Connectors/Connectors.Google.UnitTests/GeminiPromptExecutionSettingsTests.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public void ItCreatesGeminiExecutionSettingsWithCorrectDefaults()
2626
Assert.Null(executionSettings.StopSequences);
2727
Assert.Null(executionSettings.CandidateCount);
2828
Assert.Null(executionSettings.SafetySettings);
29+
Assert.Null(executionSettings.AudioTimestamp);
2930
Assert.Equal(GeminiPromptExecutionSettings.DefaultTextMaxTokens, executionSettings.MaxTokens);
3031
}
3132

@@ -39,6 +40,7 @@ public void ItUsesExistingGeminiExecutionSettings()
3940
TopP = 0.7,
4041
TopK = 20,
4142
CandidateCount = 3,
43+
AudioTimestamp = true,
4244
StopSequences = ["foo", "bar"],
4345
MaxTokens = 128,
4446
SafetySettings =
@@ -64,7 +66,8 @@ public void ItCreatesGeminiExecutionSettingsFromExtensionDataSnakeCase()
6466
ExtensionData = new Dictionary<string, object>
6567
{
6668
{ "max_tokens", 1000 },
67-
{ "temperature", 0 }
69+
{ "temperature", 0 },
70+
{ "audio_timestamp", true }
6871
}
6972
};
7073

@@ -75,6 +78,7 @@ public void ItCreatesGeminiExecutionSettingsFromExtensionDataSnakeCase()
7578
Assert.NotNull(executionSettings);
7679
Assert.Equal(1000, executionSettings.MaxTokens);
7780
Assert.Equal(0, executionSettings.Temperature);
81+
Assert.True(executionSettings.AudioTimestamp);
7882
}
7983

8084
[Fact]
@@ -91,6 +95,7 @@ public void ItCreatesGeminiExecutionSettingsFromJsonSnakeCase()
9195
"candidate_count": 2,
9296
"stop_sequences": [ "foo", "bar" ],
9397
"max_tokens": 128,
98+
"audio_timestamp": true,
9499
"safety_settings": [
95100
{
96101
"category": "{{category.Label}}",
@@ -112,6 +117,7 @@ public void ItCreatesGeminiExecutionSettingsFromJsonSnakeCase()
112117
Assert.Equal(2, executionSettings.CandidateCount);
113118
Assert.Equal(["foo", "bar"], executionSettings.StopSequences);
114119
Assert.Equal(128, executionSettings.MaxTokens);
120+
Assert.True(executionSettings.AudioTimestamp);
115121
Assert.Single(executionSettings.SafetySettings!, settings =>
116122
settings.Category.Equals(category) &&
117123
settings.Threshold.Equals(threshold));
@@ -130,6 +136,7 @@ public void PromptExecutionSettingsCloneWorksAsExpected()
130136
"top_p": 0.7,
131137
"top_k": 25,
132138
"candidate_count": 2,
139+
"audio_timestamp": true,
133140
"stop_sequences": [ "foo", "bar" ],
134141
"max_tokens": 128,
135142
"safety_settings": [
@@ -152,6 +159,7 @@ public void PromptExecutionSettingsCloneWorksAsExpected()
152159
Assert.Equivalent(executionSettings.ExtensionData, clone.ExtensionData);
153160
Assert.Equivalent(executionSettings.StopSequences, clone.StopSequences);
154161
Assert.Equivalent(executionSettings.SafetySettings, clone.SafetySettings);
162+
Assert.Equal(executionSettings.AudioTimestamp, clone.AudioTimestamp);
155163
}
156164

157165
[Fact]
@@ -167,6 +175,7 @@ public void PromptExecutionSettingsFreezeWorksAsExpected()
167175
"top_p": 0.7,
168176
"top_k": 25,
169177
"candidate_count": 2,
178+
"audio_timestamp": true,
170179
"stop_sequences": [ "foo", "bar" ],
171180
"max_tokens": 128,
172181
"safety_settings": [
@@ -187,6 +196,7 @@ public void PromptExecutionSettingsFreezeWorksAsExpected()
187196
Assert.Throws<InvalidOperationException>(() => executionSettings.ModelId = "gemini-ultra");
188197
Assert.Throws<InvalidOperationException>(() => executionSettings.CandidateCount = 5);
189198
Assert.Throws<InvalidOperationException>(() => executionSettings.Temperature = 0.5);
199+
Assert.Throws<InvalidOperationException>(() => executionSettings.AudioTimestamp = false);
190200
Assert.Throws<NotSupportedException>(() => executionSettings.StopSequences!.Add("baz"));
191201
}
192202
}

‎dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,8 @@ private static void AddConfiguration(GeminiPromptExecutionSettings executionSett
247247
TopK = executionSettings.TopK,
248248
MaxOutputTokens = executionSettings.MaxTokens,
249249
StopSequences = executionSettings.StopSequences,
250-
CandidateCount = executionSettings.CandidateCount
250+
CandidateCount = executionSettings.CandidateCount,
251+
AudioTimestamp = executionSettings.AudioTimestamp
251252
};
252253
}
253254

@@ -282,5 +283,9 @@ internal sealed class ConfigurationElement
282283
[JsonPropertyName("candidateCount")]
283284
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
284285
public int? CandidateCount { get; set; }
286+
287+
[JsonPropertyName("audioTimestamp")]
288+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
289+
public bool? AudioTimestamp { get; set; }
285290
}
286291
}

‎dotnet/src/Connectors/Connectors.Google/GeminiPromptExecutionSettings.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public sealed class GeminiPromptExecutionSettings : PromptExecutionSettings
2323
private int? _maxTokens;
2424
private int? _candidateCount;
2525
private IList<string>? _stopSequences;
26+
private bool? _audioTimestamp;
2627
private IList<GeminiSafetySetting>? _safetySettings;
2728
private GeminiToolCallBehavior? _toolCallBehavior;
2829

@@ -171,6 +172,21 @@ public GeminiToolCallBehavior? ToolCallBehavior
171172
}
172173
}
173174

175+
/// <summary>
176+
/// Indicates if the audio response should include timestamps.
177+
/// if enabled, audio timestamp will be included in the request to the model.
178+
/// </summary>
179+
[JsonPropertyName("audio_timestamp")]
180+
public bool? AudioTimestamp
181+
{
182+
get => this._audioTimestamp;
183+
set
184+
{
185+
this.ThrowIfFrozen();
186+
this._audioTimestamp = value;
187+
}
188+
}
189+
174190
/// <inheritdoc />
175191
public override void Freeze()
176192
{
@@ -207,6 +223,7 @@ public override PromptExecutionSettings Clone()
207223
StopSequences = this.StopSequences is not null ? new List<string>(this.StopSequences) : null,
208224
SafetySettings = this.SafetySettings?.Select(setting => new GeminiSafetySetting(setting)).ToList(),
209225
ToolCallBehavior = this.ToolCallBehavior?.Clone(),
226+
AudioTimestamp = this.AudioTimestamp
210227
};
211228
}
212229

0 commit comments

Comments
 (0)