Skip to content

Commit 2a741be

Browse files
steipetesteipete
authored
feat(ios): render LaTeX display math in chat via SwiftMath (#100829)
1 parent 8274022 commit 2a741be

9 files changed

Lines changed: 505 additions & 22 deletions

File tree

‎apps/ios/Resources/Licenses/SwiftMath.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
SwiftMath
2+
3+
MIT License
4+
5+
Copyright (c) 2023 Computer Inspirations
6+
7+
Permission is hereby granted, free of charge, to any person obtaining a copy
8+
of this software and associated documentation files (the "Software"), to deal
9+
in the Software without restriction, including without limitation the rights
10+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
copies of the Software, and to permit persons to whom the Software is
12+
furnished to do so, subject to the following conditions:
13+
14+
The above copyright notice and this permission notice shall be included in all
15+
copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23+
SOFTWARE.
24+
25+
SwiftMath bundles math fonts licensed separately under the GUST Font License
26+
and the SIL Open Font License.

‎apps/ios/Tests/SwiftUIRenderSmokeTests.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import SwiftUI
33
import Testing
44
import UIKit
55
@testable import OpenClaw
6+
@testable import OpenClawChatUI
67

78
struct SwiftUIRenderSmokeTests {
89
@MainActor private static func host(_ view: some View, size: CGSize? = nil) -> UIWindow {
@@ -111,6 +112,34 @@ struct SwiftUIRenderSmokeTests {
111112
}
112113
}
113114

115+
@Test @MainActor func `display math builds valid and fallback view hierarchies`() {
116+
for typeSize in [DynamicTypeSize.large, .accessibility2] {
117+
let root = VStack {
118+
ChatMathBlockView(block: ChatMathBlock(
119+
latex: #"\frac{-b \pm \sqrt{b^2 - 4ac}}{2a}"#,
120+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
121+
ChatMathBlockView(block: ChatMathBlock(
122+
latex: #"\notARealCommand{"#,
123+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
124+
ChatMathBlockView(block: ChatMathBlock(
125+
latex: "α + β = γ",
126+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
127+
ChatMathBlockView(block: ChatMathBlock(
128+
latex: String(repeating: "{", count: 65) + "x",
129+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
130+
ChatMathBlockView(block: ChatMathBlock(
131+
latex: String(repeating: #"\bar"#, count: 129) + "x",
132+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
133+
ChatMathBlockView(block: ChatMathBlock(
134+
latex: #"x\textcolor{#fff}{}"#,
135+
isComplete: true), textColor: OpenClawChatTheme.assistantText)
136+
}
137+
.environment(\.dynamicTypeSize, typeSize)
138+
139+
_ = Self.host(root, size: CGSize(width: 393, height: 240))
140+
}
141+
}
142+
114143
@Test @MainActor func `root tabs builds device orientation shell matrix`() {
115144
for scenario in Self.rootTabsShellScenarios() {
116145
let appModel = NodeAppModel()

‎apps/macos/Package.resolved‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎apps/shared/OpenClawKit/Package.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ let package = Package(
1919
],
2020
dependencies: [
2121
.package(url: "https://github.com/steipete/ElevenLabsKit", exact: "0.1.1"),
22+
.package(url: "https://github.com/mgriebling/SwiftMath", exact: "1.7.3"),
2223
.package(url: "https://github.com/swiftlang/swift-markdown", exact: "0.8.0"),
2324
],
2425
targets: [
@@ -46,6 +47,7 @@ let package = Package(
4647
dependencies: [
4748
"OpenClawKit",
4849
.product(name: "Markdown", package: "swift-markdown"),
50+
.product(name: "SwiftMath", package: "SwiftMath"),
4951
],
5052
path: "Sources/OpenClawChatUI",
5153
swiftSettings: [

‎apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockSegmenter.swift‎

Lines changed: 175 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ import Foundation
22
import Markdown
33

44
/// One renderable block of a chat message. Prose stays on the
5-
/// AttributedString pipeline; fenced code and GFM tables get dedicated views.
5+
/// AttributedString pipeline; fenced code, display math, and GFM tables get dedicated views.
66
enum ChatMarkdownBlock: Equatable {
77
case prose(String)
88
case code(ChatCodeBlock)
9+
case math(ChatMathBlock)
910
case table(ChatMarkdownTable)
1011
}
1112

@@ -17,6 +18,12 @@ struct ChatCodeBlock: Equatable {
1718
let isComplete: Bool
1819
}
1920

21+
struct ChatMathBlock: Equatable {
22+
let latex: String
23+
/// True when the delimiter was closed or the message finished streaming.
24+
let isComplete: Bool
25+
}
26+
2027
struct ChatMarkdownTable: Equatable {
2128
enum ColumnAlignment: Equatable {
2229
case leading
@@ -50,45 +57,45 @@ enum ChatMarkdownBlockSyntax {
5057
}
5158

5259
enum ChatMarkdownBlockSegmenter {
60+
static let maxMathBytes = 5000
5361
static let maxTableBytes = 20000
5462
static let maxTableRows = 100
5563
static let maxTableColumns = 12
5664
static let maxTableCells = 600
5765

58-
/// Extracts only top-level fenced code and GFM tables. The parser owns
66+
/// Extracts only top-level fenced code, display math, and GFM tables. The parser owns
5967
/// CommonMark container and reference semantics; nested blocks stay in the
6068
/// surrounding prose range unchanged.
6169
static func segments(markdown: String, isComplete: Bool) -> [ChatMarkdownBlock] {
6270
let source = SourceBuffer(markdown)
6371
let document = Document(parsing: source.markdown)
64-
var blocks: [ChatMarkdownBlock] = []
65-
var proseStart = 0
66-
67-
func appendProse(until end: Int) {
68-
guard proseStart < end else { return }
69-
blocks.append(contentsOf: self.proseOnly(Array(source.lines[proseStart..<end])))
70-
}
72+
let mathResult = self.mathExtractions(
73+
source: source,
74+
document: document,
75+
isComplete: isComplete)
76+
var extractions = mathResult.extractions
7177

7278
for child in document.children {
73-
guard let lineRange = source.lineRange(for: child.range), lineRange.lowerBound >= proseStart else {
79+
guard let lineRange = source.lineRange(for: child.range) else { continue }
80+
if mathResult.protectedRanges.contains(where: { $0.contains(lineRange.lowerBound) }) {
7481
continue
7582
}
7683

7784
if let code = child as? Markdown.CodeBlock,
7885
let opener = FenceOpener.parse(source.lines[lineRange.lowerBound])
7986
{
80-
appendProse(until: lineRange.lowerBound)
8187
let language = code.language?
8288
.split(whereSeparator: \.isWhitespace)
8389
.first
8490
.map { $0.lowercased() }
8591
let closed = lineRange.count > 1
8692
&& opener.isClose(source.lines[lineRange.index(before: lineRange.endIndex)])
87-
blocks.append(.code(ChatCodeBlock(
88-
language: language,
89-
code: self.dropStructuralCodeNewline(code.code),
90-
isComplete: closed || isComplete)))
91-
proseStart = lineRange.upperBound
93+
extractions.append(Extraction(
94+
lineRange: lineRange,
95+
block: .code(ChatCodeBlock(
96+
language: language,
97+
code: self.dropStructuralCodeNewline(code.code),
98+
isComplete: closed || isComplete))))
9299
continue
93100
}
94101

@@ -105,10 +112,29 @@ enum ChatMarkdownBlockSegmenter {
105112
{
106113
continue
107114
}
108-
appendProse(until: tableRange.lowerBound)
109-
blocks.append(.table(rendered))
110-
proseStart = tableRange.upperBound
115+
extractions.append(Extraction(lineRange: tableRange, block: .table(rendered)))
116+
}
117+
}
118+
119+
extractions.sort { left, right in
120+
if left.lineRange.lowerBound != right.lineRange.lowerBound {
121+
return left.lineRange.lowerBound < right.lineRange.lowerBound
111122
}
123+
return left.lineRange.upperBound > right.lineRange.upperBound
124+
}
125+
126+
var blocks: [ChatMarkdownBlock] = []
127+
var proseStart = 0
128+
129+
func appendProse(until end: Int) {
130+
guard proseStart < end else { return }
131+
blocks.append(contentsOf: self.proseOnly(Array(source.lines[proseStart..<end])))
132+
}
133+
134+
for extraction in extractions where extraction.lineRange.lowerBound >= proseStart {
135+
appendProse(until: extraction.lineRange.lowerBound)
136+
blocks.append(extraction.block)
137+
proseStart = extraction.lineRange.upperBound
112138
}
113139

114140
appendProse(until: source.lines.count)
@@ -118,6 +144,89 @@ enum ChatMarkdownBlockSegmenter {
118144
return blocks
119145
}
120146

147+
private static func mathExtractions(
148+
source: SourceBuffer,
149+
document: Document,
150+
isComplete: Bool) -> MathExtractionResult
151+
{
152+
guard source.markdown.contains("$$") || source.markdown.contains(#"\["#) else {
153+
return MathExtractionResult(extractions: [], protectedRanges: [])
154+
}
155+
var topLevelParagraphLines = Set<Int>()
156+
var inlineCodeLines = Set<Int>()
157+
func collectInlineCodeLines(from markup: any Markup) {
158+
if markup is Markdown.InlineCode, let lineRange = source.lineRange(for: markup.range) {
159+
inlineCodeLines.formUnion(lineRange)
160+
}
161+
for child in markup.children {
162+
collectInlineCodeLines(from: child)
163+
}
164+
}
165+
for child in document.children where child is Markdown.Paragraph {
166+
if let lineRange = source.lineRange(for: child.range) {
167+
topLevelParagraphLines.formUnion(lineRange)
168+
}
169+
collectInlineCodeLines(from: child)
170+
}
171+
var extractions: [Extraction] = []
172+
var protectedRanges: [Range<Int>] = []
173+
var lineIndex = 0
174+
175+
while lineIndex < source.lines.count {
176+
guard topLevelParagraphLines.contains(lineIndex),
177+
!inlineCodeLines.contains(lineIndex),
178+
let opener = MathDelimiter.parse(source.lines[lineIndex])
179+
else {
180+
lineIndex += 1
181+
continue
182+
}
183+
184+
if let sameLineLatex = opener.sameLineLatex {
185+
let lineRange = lineIndex..<(lineIndex + 1)
186+
if sameLineLatex.utf8.count <= self.maxMathBytes {
187+
extractions.append(Extraction(
188+
lineRange: lineRange,
189+
block: .math(ChatMathBlock(latex: sameLineLatex, isComplete: true))))
190+
} else {
191+
protectedRanges.append(lineRange)
192+
}
193+
lineIndex += 1
194+
continue
195+
}
196+
197+
let contentStart = lineIndex + 1
198+
var closeIndex = contentStart
199+
while closeIndex < source.lines.count,
200+
!opener.isClose(source.lines[closeIndex])
201+
{
202+
closeIndex += 1
203+
}
204+
205+
let closed = closeIndex < source.lines.count
206+
guard closed || isComplete else {
207+
// The first unmatched opener owns the remaining stream. Stop
208+
// here so later opener-looking lines do not trigger rescans.
209+
protectedRanges.append(lineIndex..<source.lines.count)
210+
return MathExtractionResult(extractions: extractions, protectedRanges: protectedRanges)
211+
}
212+
213+
let contentEnd = closed ? closeIndex : source.lines.count
214+
let lineRange = lineIndex..<(closed ? closeIndex + 1 : source.lines.count)
215+
let latex = source.lines[contentStart..<contentEnd]
216+
.joined(separator: "\n")
217+
.trimmingCharacters(in: .whitespacesAndNewlines)
218+
if latex.utf8.count <= self.maxMathBytes {
219+
extractions.append(Extraction(
220+
lineRange: lineRange,
221+
block: .math(ChatMathBlock(latex: latex, isComplete: closed || isComplete))))
222+
} else {
223+
protectedRanges.append(lineRange)
224+
}
225+
lineIndex = lineRange.upperBound
226+
}
227+
return MathExtractionResult(extractions: extractions, protectedRanges: protectedRanges)
228+
}
229+
121230
private static func proseOnly(_ lines: [String]) -> [ChatMarkdownBlock] {
122231
// Boundary blank lines only separate extracted blocks; the rendered
123232
// VStack provides that spacing. Interior blanks remain paragraphs.
@@ -150,6 +259,52 @@ enum ChatMarkdownBlockSegmenter {
150259
code.hasSuffix("\n") ? String(code.dropLast()) : code
151260
}
152261

262+
private struct Extraction {
263+
let lineRange: Range<Int>
264+
let block: ChatMarkdownBlock
265+
}
266+
267+
private struct MathExtractionResult {
268+
let extractions: [Extraction]
269+
/// Rejected math stays prose and owns any block-looking syntax inside its span.
270+
let protectedRanges: [Range<Int>]
271+
}
272+
273+
private struct MathDelimiter {
274+
let close: String
275+
let sameLineLatex: String?
276+
277+
static func parse(_ line: String) -> MathDelimiter? {
278+
let (indent, afterIndent) = FenceOpener.leadingSpaces(of: line)
279+
guard indent <= 3, afterIndent < line.endIndex else { return nil }
280+
let suffix = line[afterIndent...]
281+
let pair: (open: String, close: String)
282+
if suffix.hasPrefix("$$") {
283+
pair = ("$$", "$$")
284+
} else if suffix.hasPrefix(#"\["#) {
285+
pair = (#"\["#, #"\]"#)
286+
} else {
287+
return nil
288+
}
289+
290+
let contentStart = suffix.index(suffix.startIndex, offsetBy: pair.open.count)
291+
let remainder = suffix[contentStart...]
292+
if remainder.trimmingCharacters(in: .whitespaces).isEmpty {
293+
return MathDelimiter(close: pair.close, sameLineLatex: nil)
294+
}
295+
guard let closeRange = remainder.range(of: pair.close, options: .backwards),
296+
remainder[closeRange.upperBound...].trimmingCharacters(in: .whitespaces).isEmpty
297+
else { return nil }
298+
let latex = remainder[..<closeRange.lowerBound]
299+
.trimmingCharacters(in: .whitespacesAndNewlines)
300+
return MathDelimiter(close: pair.close, sameLineLatex: latex)
301+
}
302+
303+
func isClose(_ line: String) -> Bool {
304+
line.trimmingCharacters(in: .whitespaces) == self.close
305+
}
306+
}
307+
153308
private static func table(
154309
_ table: Markdown.Table,
155310
source: SourceBuffer,
@@ -324,7 +479,7 @@ enum ChatMarkdownBlockSegmenter {
324479
return count >= self.count && line[cursor...].allSatisfy(\.isWhitespace)
325480
}
326481

327-
private static func leadingSpaces(of line: String) -> (count: Int, end: String.Index) {
482+
fileprivate static func leadingSpaces(of line: String) -> (count: Int, end: String.Index) {
328483
var count = 0
329484
var cursor = line.startIndex
330485
while cursor < line.endIndex, line[cursor] == " " {

0 commit comments

Comments
 (0)