|
| 1 | +// Copyright (c) Microsoft. All rights reserved. |
| 2 | + |
| 3 | +using System.ClientModel.Primitives; |
| 4 | +using System.Text.Json; |
| 5 | +using Microsoft.Extensions.AI; |
| 6 | +using OpenAI.Responses; |
| 7 | + |
| 8 | +namespace Demo.ComputerUse; |
| 9 | + |
| 10 | +/// <summary> |
| 11 | +/// Enum for tracking the state of the simulated web search flow. |
| 12 | +/// </summary> |
| 13 | +internal enum SearchState |
| 14 | +{ |
| 15 | + Initial, // Browser search page |
| 16 | + Typed, // Text entered in search box |
| 17 | + PressedEnter // Search submitted, showing results |
| 18 | +} |
| 19 | + |
| 20 | +/// <summary> |
| 21 | +/// One action of a computer call batch, reduced to the fields this simulation needs. |
| 22 | +/// </summary> |
| 23 | +internal sealed record ComputerActionRequest(string Type, string? Text, IReadOnlyList<string> Keys); |
| 24 | + |
| 25 | +internal static class ComputerUseUtil |
| 26 | +{ |
| 27 | + internal static async Task<Dictionary<string, string>> UploadScreenshotAssetsAsync(IHostedFileClient fileClient) |
| 28 | + { |
| 29 | + string assetsDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets"); |
| 30 | + |
| 31 | + (string Key, string FileName)[] files = |
| 32 | + [ |
| 33 | + ("browser_search", "cua_browser_search.jpg"), |
| 34 | + ("search_typed", "cua_search_typed.jpg"), |
| 35 | + ("search_results", "cua_search_results.jpg") |
| 36 | + ]; |
| 37 | + |
| 38 | + Dictionary<string, string> screenshots = []; |
| 39 | + |
| 40 | + foreach ((string key, string fileName) in files) |
| 41 | + { |
| 42 | + HostedFileContent result = await fileClient.UploadAsync( |
| 43 | + Path.Combine(assetsDir, fileName), |
| 44 | + new HostedFileClientOptions { Purpose = "assistants" }); |
| 45 | + screenshots[key] = result.FileId; |
| 46 | + } |
| 47 | + |
| 48 | + return screenshots; |
| 49 | + } |
| 50 | + |
| 51 | + internal static async Task DeleteScreenshotAssetsAsync( |
| 52 | + IHostedFileClient fileClient, |
| 53 | + Dictionary<string, string> screenshots) |
| 54 | + { |
| 55 | + foreach ((string key, string fileId) in screenshots) |
| 56 | + { |
| 57 | + try |
| 58 | + { |
| 59 | + await fileClient.DeleteAsync(fileId); |
| 60 | + } |
| 61 | + catch (Exception exception) |
| 62 | + { |
| 63 | + // Continue so one failed deletion does not leave every remaining sample asset behind. |
| 64 | + Console.Error.WriteLine($"Failed to delete screenshot '{key}' ({fileId}): {exception.Message}"); |
| 65 | + } |
| 66 | + } |
| 67 | + } |
| 68 | + |
| 69 | + /// <summary> |
| 70 | + /// Reads the ordered actions of a computer call. |
| 71 | + /// </summary> |
| 72 | + /// <remarks> |
| 73 | + /// The OpenAI .NET SDK does not expose the GA <c>actions</c> batch as a typed property yet, so the call is |
| 74 | + /// re-serialized and the batch is read from its JSON. A preview call carries a single <c>action</c> instead. |
| 75 | + /// </remarks> |
| 76 | + internal static IReadOnlyList<ComputerActionRequest> GetActions(ComputerCallResponseItem computerCall) |
| 77 | + { |
| 78 | + using JsonDocument json = JsonDocument.Parse(ModelReaderWriter.Write(computerCall, ModelReaderWriterOptions.Json)); |
| 79 | + |
| 80 | + if (json.RootElement.TryGetProperty("actions", out JsonElement actions) && actions.ValueKind == JsonValueKind.Array) |
| 81 | + { |
| 82 | + return [.. actions.EnumerateArray().Select(ToActionRequest)]; |
| 83 | + } |
| 84 | + |
| 85 | + if (json.RootElement.TryGetProperty("action", out JsonElement action) && action.ValueKind == JsonValueKind.Object) |
| 86 | + { |
| 87 | + return [ToActionRequest(action)]; |
| 88 | + } |
| 89 | + |
| 90 | + return []; |
| 91 | + } |
| 92 | + |
| 93 | + /// <summary> |
| 94 | + /// Simulates executing a computer action by advancing the state. |
| 95 | + /// </summary> |
| 96 | + internal static async Task<SearchState> ApplyAsync(ComputerActionRequest action, SearchState currentState) |
| 97 | + { |
| 98 | + if (action.Type == "wait") |
| 99 | + { |
| 100 | + await Task.Delay(TimeSpan.FromSeconds(1)); |
| 101 | + } |
| 102 | + |
| 103 | + return action.Type switch |
| 104 | + { |
| 105 | + "click" when currentState == SearchState.Typed => SearchState.PressedEnter, |
| 106 | + "type" when action.Text is not null => SearchState.Typed, |
| 107 | + "keypress" when IsEnterKey(action) => SearchState.PressedEnter, |
| 108 | + _ => currentState |
| 109 | + }; |
| 110 | + } |
| 111 | + |
| 112 | + /// <summary> |
| 113 | + /// Returns the key of the screenshot that shows the given state. |
| 114 | + /// </summary> |
| 115 | + internal static string GetScreenshotKey(SearchState state) => state switch |
| 116 | + { |
| 117 | + SearchState.PressedEnter => "search_results", |
| 118 | + SearchState.Typed => "search_typed", |
| 119 | + _ => "browser_search" |
| 120 | + }; |
| 121 | + |
| 122 | + private static ComputerActionRequest ToActionRequest(JsonElement action) |
| 123 | + { |
| 124 | + string type = action.TryGetProperty("type", out JsonElement typeElement) ? typeElement.GetString() ?? string.Empty : string.Empty; |
| 125 | + string? text = action.TryGetProperty("text", out JsonElement textElement) ? textElement.GetString() : null; |
| 126 | + IReadOnlyList<string> keys = action.TryGetProperty("keys", out JsonElement keysElement) && keysElement.ValueKind == JsonValueKind.Array |
| 127 | + ? [.. keysElement.EnumerateArray().Select(k => k.GetString() ?? string.Empty)] |
| 128 | + : []; |
| 129 | + |
| 130 | + return new ComputerActionRequest(type, text, keys); |
| 131 | + } |
| 132 | + |
| 133 | + private static bool IsEnterKey(ComputerActionRequest action) => |
| 134 | + action.Keys.Contains("Return", StringComparer.OrdinalIgnoreCase) || |
| 135 | + action.Keys.Contains("Enter", StringComparer.OrdinalIgnoreCase); |
| 136 | +} |
0 commit comments