|
| 1 | +// Copyright (c) Microsoft. All rights reserved. |
| 2 | + |
| 3 | +using Microsoft.SemanticKernel; |
| 4 | +using Microsoft.SemanticKernel.ChatCompletion; |
| 5 | +using Microsoft.SemanticKernel.Connectors.OpenAI; |
| 6 | +using OpenAI.Chat; |
| 7 | + |
| 8 | +namespace Filtering; |
| 9 | + |
| 10 | +/// <summary> |
| 11 | +/// Property <see cref="OpenAIPromptExecutionSettings.MaxTokens"/> allows to specify maximum number of tokens to generate in one response. |
| 12 | +/// In Semantic Kernel, auto function calling may perform multiple requests to AI model, but with the same max tokens value. |
| 13 | +/// For example, in case when max tokens = 50, and 3 functions are expected to be called with 3 separate requests to AI model, |
| 14 | +/// each request will have max tokens = 50, which in total will result in more tokens used. |
| 15 | +/// This example shows how to limit token usage with <see cref="OpenAIPromptExecutionSettings.MaxTokens"/> property and filter |
| 16 | +/// for all requests in the same auto function calling process. |
| 17 | +/// </summary> |
| 18 | +public sealed class MaxTokensWithFilters(ITestOutputHelper output) : BaseTest(output) |
| 19 | +{ |
| 20 | + /// <summary>Output max tokens value for demonstration purposes.</summary> |
| 21 | + private const int MaxTokens = 50; |
| 22 | + |
| 23 | + [Fact] |
| 24 | + public async Task ExampleAsync() |
| 25 | + { |
| 26 | + // Run example without filter. As a result, even though max tokens = 50, it takes 83 tokens to complete |
| 27 | + // the request with auto function calling process. |
| 28 | + await this.RunExampleAsync(includeFilter: false); |
| 29 | + |
| 30 | + // Output: |
| 31 | + // Invoking MoviePlugin-GetMovieTitles function. |
| 32 | + // Invoking MoviePlugin-GetDirectors function. |
| 33 | + // Invoking MoviePlugin-GetMovieDescriptions function. |
| 34 | + // Total output tokens used: 83 |
| 35 | + |
| 36 | + // Run example with filter, which subtracts max tokens value based on previous requests. |
| 37 | + // As a result, it takes 50 tokens to complete the request, as specified in execution settings. |
| 38 | + await this.RunExampleAsync(includeFilter: true); |
| 39 | + |
| 40 | + // Output: |
| 41 | + // Invoking MoviePlugin-GetMovieTitles function. |
| 42 | + // Invoking MoviePlugin-GetDirectors function. |
| 43 | + // Invoking MoviePlugin-GetMovieDescriptions function. |
| 44 | + // Total output tokens used: 50 |
| 45 | + } |
| 46 | + |
| 47 | + #region private |
| 48 | + |
| 49 | + private async Task RunExampleAsync(bool includeFilter) |
| 50 | + { |
| 51 | + // Define execution settings with max tokens and auto function calling enabled. |
| 52 | + var executionSettings = new OpenAIPromptExecutionSettings |
| 53 | + { |
| 54 | + MaxTokens = MaxTokens, |
| 55 | + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() |
| 56 | + }; |
| 57 | + |
| 58 | + // Initialize kernel. |
| 59 | + var kernel = Kernel |
| 60 | + .CreateBuilder() |
| 61 | + .AddOpenAIChatCompletion("gpt-4", TestConfiguration.OpenAI.ApiKey) |
| 62 | + .Build(); |
| 63 | + |
| 64 | + if (includeFilter) |
| 65 | + { |
| 66 | + // Add filter to control max tokens value. |
| 67 | + kernel.AutoFunctionInvocationFilters.Add(new MaxTokensFilter(executionSettings)); |
| 68 | + } |
| 69 | + |
| 70 | + // Import plugin. |
| 71 | + kernel.ImportPluginFromObject(new MoviePlugin(this.Output)); |
| 72 | + |
| 73 | + // Get chat completion service to work with chat history. |
| 74 | + var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>(); |
| 75 | + |
| 76 | + // Initialize chat history and define a goal/prompt for function calling process. |
| 77 | + var chatHistory = new ChatHistory(); |
| 78 | + chatHistory.AddUserMessage("Get an information about movie titles, directors and descriptions."); |
| 79 | + |
| 80 | + // Get a result for defined goal/prompt. |
| 81 | + var result = await chatCompletionService.GetChatMessageContentsAsync(chatHistory, executionSettings, kernel); |
| 82 | + |
| 83 | + // Get total output tokens used for all requests to AI model during the same auto function calling process. |
| 84 | + var totalOutputTokensUsed = GetChatHistoryOutputTokens([.. result, .. chatHistory]); |
| 85 | + |
| 86 | + // Output an information about used tokens. |
| 87 | + Console.WriteLine($"Total output tokens used: {totalOutputTokensUsed}"); |
| 88 | + } |
| 89 | + |
| 90 | + /// <summary>Filter which controls max tokens value during function calling process.</summary> |
| 91 | + private sealed class MaxTokensFilter(OpenAIPromptExecutionSettings executionSettings) : IAutoFunctionInvocationFilter |
| 92 | + { |
| 93 | + public async Task OnAutoFunctionInvocationAsync(AutoFunctionInvocationContext context, Func<AutoFunctionInvocationContext, Task> next) |
| 94 | + { |
| 95 | + // Get a last assistant message with information about used tokens. |
| 96 | + var assistantMessage = context.ChatHistory.LastOrDefault(l => l.Role == AuthorRole.Assistant); |
| 97 | + |
| 98 | + // Get tokens information from metadata. |
| 99 | + var messageTokens = GetOutputTokensFromMetadata(assistantMessage?.Metadata); |
| 100 | + |
| 101 | + // Subtract a value from execution settings to use less tokens during the next request. |
| 102 | + if (messageTokens.HasValue) |
| 103 | + { |
| 104 | + executionSettings.MaxTokens -= messageTokens.Value; |
| 105 | + } |
| 106 | + |
| 107 | + // Proceed with function calling process. |
| 108 | + await next(context); |
| 109 | + } |
| 110 | + } |
| 111 | + |
| 112 | + /// <summary>Movie plugin for demonstration purposes.</summary> |
| 113 | + private sealed class MoviePlugin(ITestOutputHelper output) |
| 114 | + { |
| 115 | + [KernelFunction] |
| 116 | + public List<string> GetMovieTitles() |
| 117 | + { |
| 118 | + output.WriteLine($"Invoking {nameof(MoviePlugin)}-{nameof(GetMovieTitles)} function."); |
| 119 | + |
| 120 | + return |
| 121 | + [ |
| 122 | + "Forrest Gump", |
| 123 | + "The Sound of Music", |
| 124 | + "The Wizard of Oz", |
| 125 | + "Singin' in the Rain", |
| 126 | + "Harry Potter and the Sorcerer's Stone" |
| 127 | + ]; |
| 128 | + } |
| 129 | + |
| 130 | + [KernelFunction] |
| 131 | + public List<string> GetDirectors() |
| 132 | + { |
| 133 | + output.WriteLine($"Invoking {nameof(MoviePlugin)}-{nameof(GetDirectors)} function."); |
| 134 | + |
| 135 | + return |
| 136 | + [ |
| 137 | + "Robert Zemeckis", |
| 138 | + "Robert Wise", |
| 139 | + "Victor Fleming", |
| 140 | + "Stanley Donen and Gene Kelly", |
| 141 | + "Chris Columbus" |
| 142 | + ]; |
| 143 | + } |
| 144 | + |
| 145 | + [KernelFunction] |
| 146 | + public List<string> GetMovieDescriptions() |
| 147 | + { |
| 148 | + output.WriteLine($"Invoking {nameof(MoviePlugin)}-{nameof(GetMovieDescriptions)} function."); |
| 149 | + |
| 150 | + return |
| 151 | + [ |
| 152 | + "A heartfelt story of a man with a big heart who experiences key moments in 20th-century America.", |
| 153 | + "A young governess brings music and joy to a family in Austria.", |
| 154 | + "A young girl is swept away to a magical land and embarks on an adventurous journey home.", |
| 155 | + "A celebration of the golden age of Hollywood with iconic musical numbers.", |
| 156 | + "A young boy discovers he’s a wizard and begins his journey at Hogwarts School of Witchcraft and Wizardry." |
| 157 | + ]; |
| 158 | + } |
| 159 | + } |
| 160 | + |
| 161 | + /// <summary>Helper method to get output tokens from entire chat history.</summary> |
| 162 | + private static int GetChatHistoryOutputTokens(ChatHistory? chatHistory) |
| 163 | + { |
| 164 | + var tokens = 0; |
| 165 | + |
| 166 | + if (chatHistory is null) |
| 167 | + { |
| 168 | + return tokens; |
| 169 | + } |
| 170 | + |
| 171 | + foreach (var message in chatHistory) |
| 172 | + { |
| 173 | + var messageTokens = GetOutputTokensFromMetadata(message.Metadata); |
| 174 | + |
| 175 | + if (messageTokens.HasValue) |
| 176 | + { |
| 177 | + tokens += messageTokens.Value; |
| 178 | + } |
| 179 | + } |
| 180 | + |
| 181 | + return tokens; |
| 182 | + } |
| 183 | + |
| 184 | + /// <summary>Helper method to get output tokens from message metadata.</summary> |
| 185 | + private static int? GetOutputTokensFromMetadata(IReadOnlyDictionary<string, object?>? metadata) |
| 186 | + { |
| 187 | + if (metadata is not null && |
| 188 | + metadata.TryGetValue("Usage", out object? usageObject) && |
| 189 | + usageObject is ChatTokenUsage usage) |
| 190 | + { |
| 191 | + return usage.OutputTokenCount; |
| 192 | + } |
| 193 | + |
| 194 | + return null; |
| 195 | + } |
| 196 | + |
| 197 | + #endregion |
| 198 | +} |
0 commit comments