Skip to content

Commit a132be6

Browse files
authored
feat(swift-ios): preserve context when copying between messages (#11701)
1 parent bb18144 commit a132be6

20 files changed

Lines changed: 1477 additions & 93 deletions

‎apps/swift-ios/App/NativeFeatureClient.swift‎

Lines changed: 59 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ private struct T3ConnectManagedCleanupError: LocalizedError {
2525
@MainActor
2626
final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
2727
FeatureProjectCreationClient, FeatureWorkspaceAssetResolving, FeatureAttachmentAssetResolving,
28-
FeatureFeedbackSubmitting, T3ConnectCapable
28+
FeatureFeedbackSubmitting, FeatureContextAttachmentResolving, T3ConnectCapable
2929
{
3030
private static let maximumRetainedThreadDetails = 6
3131
private static let t3ConnectLogger = Logger(
@@ -1529,7 +1529,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
15291529
shell: shellsByEnvironmentID[environment.id]
15301530
)
15311531
let title = Self.title(from: prompt, hasAttachments: !attachments.isEmpty)
1532-
let uploads = try makeUploadAttachments(attachments)
1532+
let uploads = try await makeUploadAttachments(attachments)
15331533
if !uploads.isEmpty { _ = try await client.serverConfig() }
15341534
let runtime = coreRuntimeMode(runtimeMode)
15351535
let interaction = coreInteractionMode(interactionMode)
@@ -2208,7 +2208,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
22082208
throw NativeFeatureClientError.threadNotFound
22092209
}
22102210
let model = selection.map(coreModelSelection)
2211-
let uploads = try makeUploadAttachments(attachments)
2211+
let uploads = try await makeUploadAttachments(attachments)
22122212
if !uploads.isEmpty { _ = try await client.serverConfig() }
22132213
let runtimeMode = coreRuntimeMode(
22142214
requestedRuntimeMode ?? mapRuntimeMode(shellThread.runtimeMode)
@@ -2474,11 +2474,15 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
24742474
throw NativeFeatureClientError.inputRequestNotFound
24752475
}
24762476
let route = try threadRoute(for: request.threadID)
2477+
var uploads: [String: [UploadChatAttachment]] = [:]
2478+
for (questionID, attachments) in attachmentsByQuestionID {
2479+
uploads[questionID] = try await makeUploadAttachments(attachments)
2480+
}
24772481
_ = try await route.client.respondToUserInput(
24782482
threadID: route.wireID,
24792483
requestID: request.wireID,
24802484
answers: answers.mapValues(\.jsonValue),
2481-
attachmentsByQuestionID: try attachmentsByQuestionID.mapValues(makeUploadAttachments)
2485+
attachmentsByQuestionID: uploads
24822486
)
24832487
inputRoutes[id] = nil
24842488
removeCachedInput(id: id, threadID: route.uiID)
@@ -3433,9 +3437,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
34333437
) async throws -> FeatureUploadedAttachmentReference? {
34343438
let client = try await projectCreationClient(environmentID: environmentID)
34353439
_ = try await client.serverConfig()
3436-
let prepared = try await client.prepareAttachment(
3437-
makeUploadAttachments([attachment])[0]
3438-
)
3440+
let uploads = try await makeUploadAttachments([attachment])
3441+
let prepared = try await client.prepareAttachment(uploads[0])
34393442
return prepared.map {
34403443
FeatureUploadedAttachmentReference(
34413444
environmentID: $0.environmentID,
@@ -7097,6 +7100,25 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
70977100
return resolved.url
70987101
}
70997102

7103+
func contextAttachmentAssetURL(
7104+
environmentID: String, attachment: ComposerContextRecord.Attachment
7105+
) async throws -> URL {
7106+
try Task.checkCancellation()
7107+
guard try await runtime.environments().contains(where: { $0.id == environmentID && $0.isEnabled }) else {
7108+
throw ComposerContextClipboardError.sourceUnavailable
7109+
}
7110+
let client = try await projectCreationClient(environmentID: environmentID)
7111+
let generation = environmentGeneration
7112+
let resolved = try await client.resolvedAsset(resource: .attachment(
7113+
id: attachment.attachmentId, fileName: attachment.name, mimeType: attachment.mimeType
7114+
))
7115+
try Task.checkCancellation()
7116+
guard isKnownClient(client, environmentID: environmentID, generation: generation) else {
7117+
throw CancellationError()
7118+
}
7119+
return resolved.url
7120+
}
7121+
71007122
private func lastActivityDate(
71017123
latestUserMessageAt: String?,
71027124
latestTurn: OrchestrationLatestTurn?
@@ -7161,37 +7183,47 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
71617183

71627184
private func makeUploadAttachments(
71637185
_ attachments: [FeatureUploadAttachment]
7164-
) throws -> [UploadChatAttachment] {
7186+
) async throws -> [UploadChatAttachment] {
71657187
guard attachments.count <= 8 else {
71667188
throw NativeFeatureClientError.tooManyAttachments
71677189
}
7168-
return try attachments.map {
7169-
let reference = $0.uploadedReference.map {
7170-
UploadedAttachmentReference(
7171-
environmentID: $0.environmentID,
7172-
attachmentID: $0.attachmentID
7173-
)
7174-
}
7175-
if let ownedFile = $0.ownedFile {
7190+
let uploads = try await Task.detached(priority: .userInitiated) {
7191+
try attachments.map {
7192+
let reference = $0.uploadedReference.map {
7193+
UploadedAttachmentReference(
7194+
environmentID: $0.environmentID,
7195+
attachmentID: $0.attachmentID
7196+
)
7197+
}
7198+
if let ownedFile = $0.ownedFile {
7199+
if $0.mimeType.trimmingCharacters(in: .whitespacesAndNewlines).lowercased().hasPrefix("image/") {
7200+
return try UploadChatAttachment(
7201+
id: $0.id, data: Data(contentsOf: ownedFile.url),
7202+
name: $0.name, mimeType: $0.mimeType, uploadedReference: reference
7203+
)
7204+
}
7205+
return try UploadChatAttachment(
7206+
id: $0.id,
7207+
fileURL: ownedFile.url,
7208+
name: $0.name,
7209+
mimeType: $0.mimeType,
7210+
sizeBytes: ownedFile.byteCount,
7211+
uploadedReference: reference,
7212+
contextSource: $0.source
7213+
)
7214+
}
71767215
return try UploadChatAttachment(
71777216
id: $0.id,
7178-
fileURL: ownedFile.url,
7217+
data: $0.data,
71797218
name: $0.name,
71807219
mimeType: $0.mimeType,
7181-
sizeBytes: ownedFile.byteCount,
71827220
uploadedReference: reference,
71837221
contextSource: $0.source
71847222
)
71857223
}
7186-
return try UploadChatAttachment(
7187-
id: $0.id,
7188-
data: $0.data,
7189-
name: $0.name,
7190-
mimeType: $0.mimeType,
7191-
uploadedReference: reference,
7192-
contextSource: $0.source
7193-
)
7194-
}
7224+
}.value
7225+
try Task.checkCancellation()
7226+
return uploads
71957227
}
71967228

71977229
private func requireScope(_ scope: String, client: T3Client) async throws {

‎apps/swift-ios/Core/ComposerContext.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import Foundation
44
public struct ComposerContextRecord: Codable, Equatable, Hashable, Sendable, Identifiable {
55
public var id: String { contextId }
66
public let version: Int
7-
public let contextId: String
7+
public var contextId: String
88
public let label: String
99
public var payload: Payload
1010

@@ -103,7 +103,7 @@ public struct ComposerContextRecord: Codable, Equatable, Hashable, Sendable, Ide
103103
public let regionCount: Int?
104104
public let strokeCount: Int?
105105
public let styleChangeDetails: [StyleChange]?
106-
public let screenshotContextId: String?
106+
public var screenshotContextId: String?
107107

108108
private enum CodingKeys: String, CodingKey {
109109
case annotationId, pageUrl, pageTitle, comment, targetSummary, styleChanges
Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import Foundation
2+
3+
public struct ComposerContextClipboardFragment: Codable, Equatable, Sendable {
4+
public struct Source: Codable, Equatable, Sendable {
5+
public let environmentId: String
6+
public var threadId: String?
7+
public var messageId: String?
8+
9+
public init(environmentId: String, threadId: String? = nil, messageId: String? = nil) {
10+
self.environmentId = environmentId
11+
self.threadId = threadId
12+
self.messageId = messageId
13+
}
14+
}
15+
16+
public let version: Int
17+
public let source: Source
18+
public let records: [ComposerContextRecord]
19+
20+
public init(source: Source, records: [ComposerContextRecord]) {
21+
version = 1
22+
self.source = source
23+
self.records = records
24+
}
25+
}
26+
27+
public enum ComposerContextClipboardError: LocalizedError, Equatable {
28+
case invalidFragment
29+
case tooLarge
30+
case missingRecord
31+
case contextLimit
32+
case sourceUnavailable
33+
case draftChanged
34+
35+
public var errorDescription: String? {
36+
switch self {
37+
case .invalidFragment: "The clipboard context is invalid. Copy it again from the source."
38+
case .tooLarge: "This context is too large to copy. Select fewer items."
39+
case .missingRecord: "A copied context item is missing its saved data. Copy it again from the source."
40+
case .contextLimit: "A message can have up to 200 context items. Remove some items and paste again."
41+
case .sourceUnavailable: "Reconnect to the source environment to paste its attachments."
42+
case .draftChanged: "The draft changed while the paste was loading. Paste again."
43+
}
44+
}
45+
}
46+
47+
/// Matches the web/native clipboard contract. Binary data and resolved asset URLs stay out of this payload.
48+
public enum ComposerContextClipboard {
49+
public static let mimeType = "web application/x-t3-context-fragment+json"
50+
public static let maximumCharacters = 16_000_000
51+
public static let maximumRecords = 200
52+
53+
public struct Content: Equatable, Sendable {
54+
public let text: String
55+
public let fragment: ComposerContextClipboardFragment
56+
}
57+
58+
public static func encode(_ fragment: ComposerContextClipboardFragment) throws -> String {
59+
try validate(fragment)
60+
let data = try JSONEncoder.t3.encode(fragment)
61+
let text = String(decoding: data, as: UTF8.self)
62+
guard text.utf16.count <= maximumCharacters else { throw ComposerContextClipboardError.tooLarge }
63+
return text
64+
}
65+
66+
public static func decode(_ text: String) throws -> ComposerContextClipboardFragment {
67+
guard text.utf16.count <= maximumCharacters else { throw ComposerContextClipboardError.tooLarge }
68+
let fragment: ComposerContextClipboardFragment
69+
do {
70+
// Decode every record. An invalid record must not turn into a broken pasted link.
71+
fragment = try JSONDecoder.t3.decode(ComposerContextClipboardFragment.self, from: Data(text.utf8))
72+
} catch {
73+
throw ComposerContextClipboardError.invalidFragment
74+
}
75+
try validate(fragment)
76+
return fragment
77+
}
78+
79+
private static func validate(_ fragment: ComposerContextClipboardFragment) throws {
80+
guard fragment.version == 1, !fragment.source.environmentId.isEmpty,
81+
Set(fragment.records.map(\.contextId)).count == fragment.records.count else {
82+
throw ComposerContextClipboardError.invalidFragment
83+
}
84+
guard fragment.records.count <= maximumRecords else { throw ComposerContextClipboardError.contextLimit }
85+
guard fragment.records.allSatisfy({ record in
86+
record.label.utf16.count <= 200
87+
&& ComposerContextReferences.parseHref("t3-context://v1/\(record.kind)/\(record.contextId)") != nil
88+
}) else { throw ComposerContextClipboardError.invalidFragment }
89+
}
90+
91+
public static func html(text: String, fragment: String) -> String {
92+
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~")
93+
let encoded = fragment.addingPercentEncoding(withAllowedCharacters: allowed) ?? ""
94+
let escaped = text.replacingOccurrences(of: "&", with: "&amp;")
95+
.replacingOccurrences(of: "<", with: "&lt;").replacingOccurrences(of: ">", with: "&gt;")
96+
return "<pre data-t3-context-fragment=\"\(encoded)\">\(escaped)</pre>"
97+
}
98+
99+
public static func decodeHTML(_ html: String) throws -> ComposerContextClipboardFragment? {
100+
guard html.utf16.count <= maximumCharacters * 9 + 4_096 else {
101+
throw ComposerContextClipboardError.tooLarge
102+
}
103+
let expression = try NSRegularExpression(pattern: #"data-t3-context-fragment=["']([^"']+)["']"#)
104+
guard let match = expression.firstMatch(in: html, range: NSRange(location: 0, length: html.utf16.count)) else {
105+
return nil
106+
}
107+
guard let decoded = (html as NSString).substring(with: match.range(at: 1)).removingPercentEncoding else {
108+
throw ComposerContextClipboardError.invalidFragment
109+
}
110+
return try decode(decoded)
111+
}
112+
113+
/// Include only selected records and the screenshots required by selected annotations.
114+
public static func selected(text: String, fragment: ComposerContextClipboardFragment) throws -> Content {
115+
try validate(fragment)
116+
let references = ComposerContextReferences.collect(text)
117+
let byID = Dictionary(uniqueKeysWithValues: fragment.records.map { ($0.contextId, $0) })
118+
var ids = Set<String>()
119+
for reference in references {
120+
guard let record = byID[reference.contextId], record.kind == reference.kind else {
121+
throw ComposerContextClipboardError.missingRecord
122+
}
123+
ids.insert(record.contextId)
124+
if case let .previewAnnotation(annotation) = record.payload, let screenshot = annotation.screenshotContextId {
125+
guard byID[screenshot]?.kind == "image" else { throw ComposerContextClipboardError.missingRecord }
126+
ids.insert(screenshot)
127+
}
128+
}
129+
return Content(text: text, fragment: .init(source: fragment.source, records: fragment.records.filter { ids.contains($0.contextId) }))
130+
}
131+
132+
public static func reidentified(_ content: Content, createID: () -> String = { UUID().uuidString }) throws -> Content {
133+
let selected = try selected(text: content.text, fragment: content.fragment)
134+
let ids = Dictionary(uniqueKeysWithValues: selected.fragment.records.map { ($0.contextId, createID()) })
135+
guard Set(ids.values).count == ids.count else { throw ComposerContextClipboardError.invalidFragment }
136+
let text = ComposerContextReferences.replace(selected.text) { reference in
137+
let original = (selected.text as NSString).substring(with: reference.range)
138+
return original.replacingOccurrences(
139+
of: "t3-context://v1/\(reference.kind)/\(reference.contextId)",
140+
with: "t3-context://v1/\(reference.kind)/\(ids[reference.contextId]!)"
141+
)
142+
}
143+
let records = selected.fragment.records.map { record in
144+
var result = record
145+
result.contextId = ids[record.contextId]!
146+
if case var .previewAnnotation(annotation) = record.payload, let screenshot = annotation.screenshotContextId {
147+
annotation.screenshotContextId = ids[screenshot]!
148+
result.payload = .previewAnnotation(annotation)
149+
}
150+
return result
151+
}
152+
return Content(text: text, fragment: .init(source: selected.fragment.source, records: records))
153+
}
154+
}

‎apps/swift-ios/Features/Chat/FeatureComposerTextInput.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ struct FeatureComposerTextInput: UIViewRepresentable {
2626
var onPasteTextAttachment: ((String, @escaping @MainActor () -> Bool) -> Void)? = nil
2727
var onPasteTextError: ((String) -> Void)? = nil
2828
var draftOwnerID: String = ""
29+
var onCopyContext: ((String) throws -> Bool)? = nil
30+
var onPasteContext: ((ComposerContextClipboard.Content, String, NSRange) -> Void)? = nil
2931

3032
func makeCoordinator() -> Coordinator {
3133
Coordinator(self)
@@ -42,6 +44,11 @@ struct FeatureComposerTextInput: UIViewRepresentable {
4244
textView.onPasteTextAttachment = onPasteTextAttachment
4345
textView.draftOwnerID = draftOwnerID
4446
textView.onPasteTextError = onPasteTextError
47+
textView.onCopySelection = { selected in
48+
try onCopyContext?(FeatureInlineSkillProjection.plainText(from: selected)) ?? false
49+
}
50+
textView.onCopyError = onPasteTextError
51+
textView.onPasteContext = onPasteContext
4552
if onDismissKeyboard != nil {
4653
textView.installDismissPanRecognizer()
4754
}
@@ -76,6 +83,11 @@ struct FeatureComposerTextInput: UIViewRepresentable {
7683
textView.onPasteTextAttachment = onPasteTextAttachment
7784
textView.draftOwnerID = draftOwnerID
7885
textView.onPasteTextError = onPasteTextError
86+
textView.onCopySelection = { selected in
87+
try onCopyContext?(FeatureInlineSkillProjection.plainText(from: selected)) ?? false
88+
}
89+
textView.onCopyError = onPasteTextError
90+
textView.onPasteContext = onPasteContext
7991
textView.isReadOnly = isReadOnly
8092

8193
let previousAttributedText = textView.attributedText ?? NSAttributedString()
@@ -488,6 +500,7 @@ final class FeatureComposerUITextView: FeatureInlineSkillTextView {
488500
var draftOwnerID = ""
489501
private var pastedTextRequestID = UUID()
490502
var onPasteTextError: ((String) -> Void)?
503+
var onPasteContext: ((ComposerContextClipboard.Content, String, NSRange) -> Void)?
491504
private var wantsFirstResponderOnAttach = false
492505

493506
override var keyCommands: [UIKeyCommand]? {
@@ -625,6 +638,22 @@ final class FeatureComposerUITextView: FeatureInlineSkillTextView {
625638
// attached screenshot reads as a bug.
626639
override func paste(_ sender: Any?) {
627640
guard !isReadOnly else { return }
641+
do {
642+
if let content = try FeatureContextClipboard.read() {
643+
guard let onPasteContext, markedTextRange == nil else {
644+
throw ComposerContextClipboardError.draftChanged
645+
}
646+
onPasteContext(
647+
content,
648+
FeatureInlineSkillProjection.plainText(from: attributedText),
649+
FeatureInlineSkillProjection.plainRange(for: selectedRange, in: attributedText)
650+
)
651+
return
652+
}
653+
} catch {
654+
onPasteTextError?(error.localizedDescription)
655+
return
656+
}
628657
if acceptsImages {
629658
let imageProviders = UIPasteboard.general.itemProviders.filter {
630659
$0.hasItemConformingToTypeIdentifier(UTType.image.identifier)

0 commit comments

Comments
 (0)