Skip to content

Commit 70fb34e

Browse files
committed
feat(opencode): pre-filter messages by --since/--until in loader
Previously the opencode loader read all SQLite rows and JSON files regardless of --since/--until, relying solely on post-load filtering. On large installs this caused the command to hang. Pre-filter both the SQLite row loop and JSON file loop by extracting the timestamp from the raw JSON before full deserialization, skipping entries outside the date range early.
1 parent e23d8fa commit 70fb34e

2 files changed

Lines changed: 174 additions & 14 deletions

File tree

‎.github/workflows/pullfrog.yml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,36 +22,35 @@ jobs:
2222
contents: read
2323
steps:
2424
- name: Checkout code
25-
uses: actions/checkout@v6
25+
uses: actions/checkout@v6 # zizmor: ignore[unpinned-uses]
2626
with:
2727
fetch-depth: 1
2828
- name: Setup Nix
2929
uses: ./.github/actions/setup-nix
3030
- name: Run agent
31-
uses: pullfrog/pullfrog@v0
31+
uses: pullfrog/pullfrog@v0 # zizmor: ignore[unpinned-uses]
3232
with:
3333
prompt: ${{ inputs.prompt }}
3434
env:
3535
# add at least one provider API key
3636
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
3737
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
3838
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
39-
GOOGLE_GENERATIVE_AI_API_KEY:
40-
${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
39+
GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
4140
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
4241
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
4342
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
4443
MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
4544
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
4645
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
47-
46+
4847
# for Amazon Bedrock (https://docs.pullfrog.com/bedrock)
4948
# AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
5049
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
5150
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
5251
# AWS_REGION: us-east-1
5352
# BEDROCK_MODEL_ID: <bedrock-model-id>
54-
53+
5554
# for Google Vertex AI (https://docs.pullfrog.com/vertex)
5655
# VERTEX_SERVICE_ACCOUNT_JSON: ${{ secrets.VERTEX_SERVICE_ACCOUNT_JSON }}
5756
# GOOGLE_CLOUD_PROJECT: my-project

‎rust/crates/ccusage/src/adapter/opencode/loader.rs‎

Lines changed: 169 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ use serde_json::Value;
1010
use super::{parser::message_value_to_entry, paths::paths};
1111
use crate::{
1212
cli::{CostMode, SharedArgs},
13-
collect_files_with_extension, debug_log, parse_tz, LoadedEntry, PricingMap, Result,
13+
collect_files_with_extension, debug_log, format_date_tz, parse_tz, LoadedEntry, PricingMap,
14+
Result, TimestampMs,
1415
};
1516

1617
pub(crate) fn load_entries(shared: &SharedArgs) -> Result<Vec<LoadedEntry>> {
@@ -70,7 +71,9 @@ pub(crate) fn load_entries_from_directory(
7071
let mut files = Vec::new();
7172
collect_files_with_extension(&messages_dir, "json", &mut files);
7273
for file in files {
73-
if let Some(entry) = read_message_file(&file, tz.as_ref(), shared.mode, pricing.as_ref())? {
74+
if let Some(entry) =
75+
read_message_file(&file, tz.as_ref(), shared.mode, pricing.as_ref(), shared)
76+
{
7477
if let Some(id) = entry_id(&entry) {
7578
if !seen.insert(id.to_string()) {
7679
continue;
@@ -148,6 +151,11 @@ fn load_entries_from_database(
148151
let Ok(data) = statement.read::<String, _>(2) else {
149152
continue;
150153
};
154+
if let Some(millis) = extract_message_timestamp(&data) {
155+
if !timestamp_within_range(millis, tz, shared) {
156+
continue;
157+
}
158+
}
151159
let Ok(value) = serde_json::from_str::<Value>(&data) else {
152160
continue;
153161
};
@@ -175,20 +183,47 @@ fn read_message_file(
175183
tz: Option<&JiffTimeZone>,
176184
mode: CostMode,
177185
pricing: Option<&PricingMap>,
178-
) -> Result<Option<LoadedEntry>> {
179-
let content = fs::read_to_string(path)?;
186+
shared: &SharedArgs,
187+
) -> Option<LoadedEntry> {
188+
let content = std::fs::read_to_string(path).ok()?;
189+
if let Some(millis) = extract_message_timestamp(&content) {
190+
if !timestamp_within_range(millis, tz, shared) {
191+
return None;
192+
}
193+
}
180194
let Ok(value) = serde_json::from_str::<Value>(&content) else {
181-
return Ok(None);
195+
return None;
182196
};
183-
Ok(message_value_to_entry(
184-
&value, None, None, tz, mode, pricing,
185-
))
197+
message_value_to_entry(&value, None, None, tz, mode, pricing)
186198
}
187199

188200
fn entry_id(entry: &LoadedEntry) -> Option<&str> {
189201
entry.data.message.id.as_deref().filter(|id| !id.is_empty())
190202
}
191203

204+
fn extract_message_timestamp(data: &str) -> Option<i64> {
205+
let start = data.find("\"created\":")?;
206+
let after_key = data[start + "\"created\":".len()..].trim_start();
207+
let end = after_key.find(|c: char| !c.is_ascii_digit())?;
208+
after_key[..end].parse::<i64>().ok()
209+
}
210+
211+
fn timestamp_within_range(millis: i64, tz: Option<&JiffTimeZone>, shared: &SharedArgs) -> bool {
212+
if shared.since.is_none() && shared.until.is_none() {
213+
return true;
214+
}
215+
let timestamp = TimestampMs::from_millis(millis);
216+
let date = format_date_tz(timestamp, tz).replace('-', "");
217+
shared
218+
.since
219+
.as_ref()
220+
.is_none_or(|since| date.as_str() >= since.as_str())
221+
&& shared
222+
.until
223+
.as_ref()
224+
.is_none_or(|until| date.as_str() <= until.as_str())
225+
}
226+
192227
#[cfg(test)]
193228
mod tests {
194229
use std::path::Path;
@@ -311,4 +346,130 @@ mod tests {
311346
assert_eq!(entries[0].data.message.usage.input_tokens, 120);
312347
assert_eq!(entries[0].cost, 0.03);
313348
}
349+
350+
#[test]
351+
fn filters_sqlite_entries_by_since() {
352+
let fixture = fs_fixture!({});
353+
create_db_message(
354+
&fixture.path("opencode.db"),
355+
"db-msg-1",
356+
"db-session-a",
357+
r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60}}"#,
358+
);
359+
360+
let shared = SharedArgs {
361+
mode: CostMode::Display,
362+
timezone: Some("UTC".to_string()),
363+
since: Some("20260103".to_string()),
364+
..SharedArgs::default()
365+
};
366+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
367+
assert!(
368+
entries.is_empty(),
369+
"message on 2026-01-02 should be excluded by since=20260103"
370+
);
371+
}
372+
373+
#[test]
374+
fn filters_sqlite_entries_by_until() {
375+
let fixture = fs_fixture!({});
376+
create_db_message(
377+
&fixture.path("opencode.db"),
378+
"db-msg-1",
379+
"db-session-a",
380+
r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60}}"#,
381+
);
382+
383+
let shared = SharedArgs {
384+
mode: CostMode::Display,
385+
timezone: Some("UTC".to_string()),
386+
until: Some("20260101".to_string()),
387+
..SharedArgs::default()
388+
};
389+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
390+
assert!(
391+
entries.is_empty(),
392+
"message on 2026-01-02 should be excluded by until=20260101"
393+
);
394+
}
395+
396+
#[test]
397+
fn filters_json_file_entries_by_since() {
398+
let fixture = fs_fixture!({
399+
"storage/message/message.json": r#"{"id":"msg-1","sessionID":"session-a","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":100,"output":50},"cost":0.02}"#,
400+
});
401+
402+
let shared = SharedArgs {
403+
mode: CostMode::Display,
404+
timezone: Some("UTC".to_string()),
405+
since: Some("20260103".to_string()),
406+
..SharedArgs::default()
407+
};
408+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
409+
assert!(
410+
entries.is_empty(),
411+
"message on 2026-01-02 should be excluded by since=20260103"
412+
);
413+
}
414+
415+
#[test]
416+
fn includes_entries_when_since_until_bracket_date() {
417+
let fixture = fs_fixture!({
418+
"storage/message/message.json": r#"{"id":"msg-1","sessionID":"session-a","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":100,"output":50},"cost":0.02}"#,
419+
});
420+
421+
let shared = SharedArgs {
422+
mode: CostMode::Display,
423+
timezone: Some("UTC".to_string()),
424+
since: Some("20260101".to_string()),
425+
until: Some("20260103".to_string()),
426+
..SharedArgs::default()
427+
};
428+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
429+
assert_eq!(
430+
entries.len(),
431+
1,
432+
"message on 2026-01-02 should be included when since=20260101 and until=20260103"
433+
);
434+
}
435+
436+
#[test]
437+
fn includes_entries_when_since_exact_match() {
438+
let fixture = fs_fixture!({
439+
"storage/message/message.json": r#"{"id":"msg-1","sessionID":"session-a","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":100,"output":50},"cost":0.02}"#,
440+
});
441+
442+
let shared = SharedArgs {
443+
mode: CostMode::Display,
444+
timezone: Some("UTC".to_string()),
445+
since: Some("20260102".to_string()),
446+
..SharedArgs::default()
447+
};
448+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
449+
assert_eq!(
450+
entries.len(),
451+
1,
452+
"message on 2026-01-02 should be included when since=20260102"
453+
);
454+
}
455+
456+
#[test]
457+
fn includes_entries_when_until_exact_match() {
458+
let fixture = fs_fixture!({
459+
"storage/message/message.json": r#"{"id":"msg-1","sessionID":"session-a","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":100,"output":50},"cost":0.02}"#,
460+
});
461+
462+
let shared = SharedArgs {
463+
mode: CostMode::Display,
464+
timezone: Some("UTC".to_string()),
465+
until: Some("20260102".to_string()),
466+
..SharedArgs::default()
467+
};
468+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
469+
assert_eq!(
470+
entries.len(),
471+
1,
472+
"message on 2026-01-02 should be included when until=20260102"
473+
);
474+
}
314475
}

0 commit comments

Comments
 (0)