@@ -29,7 +29,7 @@ import * as Layer from "effect/Layer";
2929import * as Scheduler from "effect/Scheduler" ;
3030import * as Schema from "effect/Schema" ;
3131import * as TestClock from "effect/testing/TestClock" ;
32- import { HttpClient , HttpClientResponse } from "effect/http" ;
32+ import { HttpClient , HttpClientError , HttpClientRequest , HttpClientResponse } from "effect/http" ;
3333
3434import * as ServerConfig from "../config.ts" ;
3535import * as ServerSettings from "../serverSettings.ts" ;
@@ -115,8 +115,12 @@ const layerService = (input: {
115115 readonly prefix : string ;
116116 readonly home : string ;
117117 readonly settings : Parameters < typeof ServerSettings . layerTest > [ 0 ] ;
118- readonly onRatesFetch ?: ( ) => void ;
118+ readonly onRatesFetch ?: ( request : HttpClientRequest . HttpClientRequest ) => void ;
119119 readonly ratesDocument ?: unknown ;
120+ readonly ratesStatus ?: ( ) => number ;
121+ readonly ratesResponse ?: (
122+ request : HttpClientRequest . HttpClientRequest ,
123+ ) => Effect . Effect < Response , HttpClientError . HttpClientError > ;
120124 readonly environment ?: NodeJS . ProcessEnv ;
121125 readonly platform ?: NodeJS . Platform ;
122126} ) =>
@@ -137,9 +141,18 @@ const layerService = (input: {
137141 Layer . succeed (
138142 HttpClient . HttpClient ,
139143 HttpClient . make ( ( request ) =>
140- Effect . sync ( ( ) => {
141- input . onRatesFetch ?.( ) ;
142- return HttpClientResponse . fromWeb ( request , Response . json ( input . ratesDocument ?? { } ) ) ;
144+ Effect . suspend ( ( ) => {
145+ input . onRatesFetch ?.( request ) ;
146+ const response = input . ratesResponse
147+ ? input . ratesResponse ( request )
148+ : Effect . succeed (
149+ Response . json ( input . ratesDocument ?? { } , {
150+ status : input . ratesStatus ?.( ) ?? 200 ,
151+ } ) ,
152+ ) ;
153+ return response . pipe (
154+ Effect . map ( ( response ) => HttpClientResponse . fromWeb ( request , response ) ) ,
155+ ) ;
143156 } ) ,
144157 ) ,
145158 ) ,
@@ -1834,10 +1847,10 @@ describe("UsageService", () => {
18341847 { concurrency : 2 } ,
18351848 ) ;
18361849 assert . deepStrictEqual ( first , second ) ;
1837- assert . strictEqual ( ratesFetches , 1 ) ;
1850+ assert . strictEqual ( ratesFetches , 2 ) ;
18381851
18391852 yield * service . readSummary ( WINDOW ) ;
1840- assert . strictEqual ( ratesFetches , 2 ) ;
1853+ assert . strictEqual ( ratesFetches , 4 ) ;
18411854 } ) . pipe ( Effect . scoped ) ,
18421855 ) ;
18431856
@@ -1880,6 +1893,149 @@ describe("UsageService", () => {
18801893 } ) . pipe ( Effect . scoped , Effect . provide ( TestClock . layer ( ) ) ) ,
18811894 ) ;
18821895
1896+ it . live ( "preserves cached estimates when both pricing downloads fail and recovers" , ( ) =>
1897+ Effect . gen ( function * ( ) {
1898+ const { transcript, settings, home } = yield * setup ;
1899+ yield * Effect . promise ( ( ) => NodeFSP . writeFile ( transcript , claudeLine ( 1 , 5 ) ) ) ;
1900+ let status = 200 ;
1901+ const requests : string [ ] = [ ] ;
1902+ const service = yield * UsageService . make . pipe (
1903+ Effect . provide (
1904+ layerService ( {
1905+ prefix : "usage-service-rates-failure-test" ,
1906+ home,
1907+ settings,
1908+ ratesDocument : {
1909+ "claude-fable-5" : { input_cost_per_token : 1e-5 , output_cost_per_token : 5e-5 } ,
1910+ } ,
1911+ ratesStatus : ( ) => status ,
1912+ onRatesFetch : ( request ) => requests . push ( request . url ) ,
1913+ } ) ,
1914+ ) ,
1915+ ) ;
1916+ const first = yield * service . readSummary ( WINDOW ) ;
1917+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
1918+ status = 503 ;
1919+
1920+ const failed = yield * service . refreshRates ;
1921+ assert . strictEqual ( failed . status , "cached" ) ;
1922+ assert . strictEqual ( failed . fetchedAt , first . pricing . fetchedAt ) ;
1923+ assert . strictEqual ( requests . length , 3 ) ;
1924+ assert . strictEqual (
1925+ requests [ 2 ] ,
1926+ "https://api.github.com/repos/BerriAI/litellm/contents/model_prices_and_context_window.json?ref=main" ,
1927+ ) ;
1928+ const cached = yield * service . readSummary ( WINDOW ) ;
1929+ assert . deepStrictEqual ( cached . buckets , first . buckets ) ;
1930+
1931+ status = 200 ;
1932+ const recovered = yield * service . refreshRates ;
1933+ assert . strictEqual ( recovered . status , "fresh" ) ;
1934+ assert . notStrictEqual ( recovered . fetchedAt , first . pricing . fetchedAt ) ;
1935+ } ) . pipe ( Effect . scoped , Effect . provide ( TestClock . layer ( ) ) ) ,
1936+ ) ;
1937+
1938+ it . live . each ( [ "network" , "http" , "invalidJson" , "empty" ] as const ) (
1939+ "recovers pricing through the GitHub API after a %s failure" ,
1940+ ( failure ) =>
1941+ Effect . gen ( function * ( ) {
1942+ const { transcript, settings, home } = yield * setup ;
1943+ yield * Effect . promise ( ( ) => NodeFSP . writeFile ( transcript , claudeLine ( 1 , 5 ) ) ) ;
1944+ const requests : HttpClientRequest . HttpClientRequest [ ] = [ ] ;
1945+ const ratesDocument = {
1946+ "claude-fable-5" : { input_cost_per_token : 1e-5 , output_cost_per_token : 5e-5 } ,
1947+ } ;
1948+ const serviceLayer = layerService ( {
1949+ prefix : `usage-service-rates-fallback-${ failure } -test` ,
1950+ home,
1951+ settings,
1952+ onRatesFetch : ( request ) => requests . push ( request ) ,
1953+ ratesResponse : ( request ) => {
1954+ if ( request . url . startsWith ( "https://api.github.com/" ) ) {
1955+ return Effect . succeed ( Response . json ( ratesDocument ) ) ;
1956+ }
1957+ if ( failure === "network" ) {
1958+ return Effect . fail (
1959+ new HttpClientError . HttpClientError ( {
1960+ reason : new HttpClientError . TransportError ( {
1961+ request,
1962+ cause : new Error ( "getaddrinfo ENOTFOUND raw.githubusercontent.com" ) ,
1963+ } ) ,
1964+ } ) ,
1965+ ) ;
1966+ }
1967+ return Effect . succeed (
1968+ failure === "invalidJson"
1969+ ? new Response ( "not JSON" )
1970+ : Response . json ( { } , { status : failure === "http" ? 503 : 200 } ) ,
1971+ ) ;
1972+ } ,
1973+ } ) ;
1974+ const service = yield * UsageService . make . pipe ( Effect . provide ( serviceLayer ) ) ;
1975+
1976+ const summary = yield * service . readSummary ( WINDOW ) ;
1977+ assert . strictEqual ( summary . pricing . status , "fresh" ) ;
1978+ assert . strictEqual ( summary . pricing . knownModels , 1 ) ;
1979+ assert . closeTo ( summary . buckets [ 0 ] ?. costUsd ?? - 1 , 0.00035 , 1e-12 ) ;
1980+ assert . deepStrictEqual (
1981+ requests . map ( ( request ) => request . url ) ,
1982+ [
1983+ "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" ,
1984+ "https://api.github.com/repos/BerriAI/litellm/contents/model_prices_and_context_window.json?ref=main" ,
1985+ ] ,
1986+ ) ;
1987+ assert . strictEqual ( requests [ 1 ] ?. headers . accept , "application/vnd.github.raw+json" ) ;
1988+
1989+ const restarted = yield * UsageService . make . pipe ( Effect . provide ( serviceLayer ) ) ;
1990+ const cached = yield * restarted . readSummary ( WINDOW ) ;
1991+ assert . strictEqual ( cached . pricing . status , "cached" ) ;
1992+ assert . deepStrictEqual ( cached . buckets , summary . buckets ) ;
1993+ assert . strictEqual ( requests . length , 2 ) ;
1994+ } ) . pipe ( Effect . scoped ) ,
1995+ ) ;
1996+
1997+ it . live ( "tries the GitHub API after the primary pricing request times out" , ( ) =>
1998+ Effect . gen ( function * ( ) {
1999+ const { settings, home } = yield * setup ;
2000+ const primaryStarted = yield * Deferred . make < void > ( ) ;
2001+ const service = yield * UsageService . make . pipe (
2002+ Effect . provide (
2003+ layerService ( {
2004+ prefix : "usage-service-rates-timeout-test" ,
2005+ home,
2006+ settings,
2007+ ratesResponse : ( request ) =>
2008+ request . url . startsWith ( "https://api.github.com/" )
2009+ ? Effect . succeed (
2010+ Response . json ( {
2011+ "claude-fable-5" : { input_cost_per_token : 1e-5 , output_cost_per_token : 5e-5 } ,
2012+ } ) ,
2013+ )
2014+ : Deferred . succeed ( primaryStarted , undefined ) . pipe ( Effect . andThen ( Effect . never ) ) ,
2015+ } ) ,
2016+ ) ,
2017+ ) ;
2018+ const refresh = yield * service . refreshRates . pipe ( Effect . forkChild ) ;
2019+ yield * Deferred . await ( primaryStarted ) ;
2020+ yield * TestClock . adjust ( Duration . seconds ( 10 ) ) ;
2021+ const pricing = yield * Fiber . join ( refresh ) ;
2022+ assert . strictEqual ( pricing . status , "fresh" ) ;
2023+ assert . strictEqual ( pricing . knownModels , 1 ) ;
2024+ } ) . pipe ( Effect . scoped , Effect . provide ( TestClock . layer ( ) ) ) ,
2025+ ) ;
2026+
2027+ it . live ( "keeps pricing unavailable when neither download has usable rates" , ( ) =>
2028+ Effect . gen ( function * ( ) {
2029+ const { settings, home } = yield * setup ;
2030+ const service = yield * UsageService . make . pipe (
2031+ Effect . provide ( layerService ( { prefix : "usage-service-rates-empty-test" , home, settings } ) ) ,
2032+ ) ;
2033+ const pricing = yield * service . refreshRates ;
2034+ assert . strictEqual ( pricing . status , "unavailable" ) ;
2035+ assert . isNull ( pricing . fetchedAt ) ;
2036+ } ) . pipe ( Effect . scoped ) ,
2037+ ) ;
2038+
18832039 it . live ( "does not orphan an in-flight scan when its first caller is interrupted" , ( ) =>
18842040 Effect . gen ( function * ( ) {
18852041 const { settings, home } = yield * setup ;
0 commit comments