Skip to content

Commit b26b2b7

Browse files
bgwbgw
authored
Turbopack: Add a custom serde_self_describing Serializer/Deserializer (#86885)
Implements a minimal custom serde `Serializer` and `Deserializer` on top of bincode with a custom self-describing format. The API is exposed as functions that can be used in `#[bincode(with = "...")]` attributes. This doesn't encode as efficiently as `pot`, but it lets us get rid of that dependency entirely. It also gives us a more efficient option than `serde_json` for certain values.
1 parent aaee236 commit b26b2b7

File tree

14 files changed

+1197
-111
lines changed

14 files changed

+1197
-111
lines changed

‎Cargo.lock‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/next-core/src/next_config.rs‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ pub struct NextConfig {
9292
cache_handler: Option<RcStr>,
9393
#[bincode(with_serde)]
9494
cache_handlers: Option<FxIndexMap<RcStr, RcStr>>,
95-
#[bincode(with = "turbo_bincode::serde_json")]
95+
#[bincode(with = "turbo_bincode::serde_self_describing")]
9696
env: FxIndexMap<String, JsonValue>,
9797
experimental: ExperimentalConfig,
9898
images: ImageConfig,
@@ -101,12 +101,12 @@ pub struct NextConfig {
101101
react_production_profiling: Option<bool>,
102102
react_strict_mode: Option<bool>,
103103
transpile_packages: Option<Vec<RcStr>>,
104-
#[bincode(with = "turbo_bincode::serde_json")]
104+
#[bincode(with = "turbo_bincode::serde_self_describing")]
105105
modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
106106
dist_dir: RcStr,
107107
dist_dir_root: RcStr,
108108
deployment_id: Option<RcStr>,
109-
#[bincode(with = "turbo_bincode::serde_json")]
109+
#[bincode(with = "turbo_bincode::serde_self_describing")]
110110
sass_options: Option<serde_json::Value>,
111111
trailing_slash: Option<bool>,
112112
asset_prefix: Option<RcStr>,
@@ -119,9 +119,9 @@ pub struct NextConfig {
119119
output: Option<OutputType>,
120120
turbopack: Option<TurbopackConfig>,
121121
production_browser_source_maps: bool,
122-
#[bincode(with = "turbo_bincode::serde_json")]
122+
#[bincode(with = "turbo_bincode::serde_self_describing")]
123123
output_file_tracing_includes: Option<serde_json::Value>,
124-
#[bincode(with = "turbo_bincode::serde_json")]
124+
#[bincode(with = "turbo_bincode::serde_self_describing")]
125125
output_file_tracing_excludes: Option<serde_json::Value>,
126126
// TODO: This option is not respected, it uses Turbopack's root instead.
127127
output_file_tracing_root: Option<RcStr>,
@@ -154,9 +154,9 @@ pub struct NextConfig {
154154
http_agent_options: HttpAgentConfig,
155155
on_demand_entries: OnDemandEntriesConfig,
156156
powered_by_header: bool,
157-
#[bincode(with = "turbo_bincode::serde_json")]
157+
#[bincode(with = "turbo_bincode::serde_self_describing")]
158158
public_runtime_config: FxIndexMap<String, serde_json::Value>,
159-
#[bincode(with = "turbo_bincode::serde_json")]
159+
#[bincode(with = "turbo_bincode::serde_self_describing")]
160160
server_runtime_config: FxIndexMap<String, serde_json::Value>,
161161
static_page_generation_timeout: f64,
162162
target: Option<String>,
@@ -686,9 +686,9 @@ pub enum RemotePatternProtocol {
686686
)]
687687
#[serde(rename_all = "camelCase")]
688688
pub struct TurbopackConfig {
689-
#[bincode(with = "turbo_bincode::serde_json")]
689+
#[bincode(with = "turbo_bincode::serde_self_describing")]
690690
pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>,
691-
#[bincode(with = "turbo_bincode::serde_json")]
691+
#[bincode(with = "turbo_bincode::serde_self_describing")]
692692
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
693693
pub resolve_extensions: Option<Vec<RcStr>>,
694694
pub debug_ids: Option<bool>,
@@ -955,7 +955,7 @@ pub struct ExperimentalConfig {
955955
/// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs)
956956
mdx_rs: Option<MdxRsOptions>,
957957
strict_next_head: Option<bool>,
958-
#[bincode(with = "turbo_bincode::serde_json")]
958+
#[bincode(with = "turbo_bincode::serde_self_describing")]
959959
swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
960960
external_middleware_rewrites_resolve: Option<bool>,
961961
scroll_restoration: Option<bool>,
@@ -964,7 +964,7 @@ pub struct ExperimentalConfig {
964964
middleware_prefetch: Option<MiddlewarePrefetchType>,
965965
/// optimizeCss can be boolean or critters' option object
966966
/// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts))
967-
#[bincode(with = "turbo_bincode::serde_json")]
967+
#[bincode(with = "turbo_bincode::serde_self_describing")]
968968
optimize_css: Option<serde_json::Value>,
969969
next_script_workers: Option<bool>,
970970
web_vitals_attribution: Option<Vec<RcStr>>,
@@ -982,15 +982,15 @@ pub struct ExperimentalConfig {
982982
adjust_font_fallbacks_with_size_adjust: Option<bool>,
983983
after: Option<bool>,
984984
app_document_preloading: Option<bool>,
985-
#[bincode(with = "turbo_bincode::serde_json")]
985+
#[bincode(with = "turbo_bincode::serde_self_describing")]
986986
cache_life: Option<FxIndexMap<String, CacheLifeProfile>>,
987987
case_sensitive_routes: Option<bool>,
988988
cpus: Option<f64>,
989989
cra_compat: Option<bool>,
990990
disable_optimized_loading: Option<bool>,
991991
disable_postcss_preset_env: Option<bool>,
992992
esm_externals: Option<EsmExternals>,
993-
#[bincode(with = "turbo_bincode::serde_json")]
993+
#[bincode(with = "turbo_bincode::serde_self_describing")]
994994
extension_alias: Option<serde_json::Value>,
995995
external_dir: Option<bool>,
996996
/// If set to `false`, webpack won't fall back to polyfill Node.js modules
@@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig {
10051005
instrumentation_hook: Option<bool>,
10061006
client_trace_metadata: Option<Vec<String>>,
10071007
large_page_data_bytes: Option<f64>,
1008-
#[bincode(with = "turbo_bincode::serde_json")]
1008+
#[bincode(with = "turbo_bincode::serde_self_describing")]
10091009
logging: Option<serde_json::Value>,
10101010
memory_based_workers_count: Option<bool>,
10111011
/// Optimize React APIs for server builds.
@@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig {
10241024
/// @internal Used by the Next.js internals only.
10251025
trust_host_header: Option<bool>,
10261026

1027-
#[bincode(with = "turbo_bincode::serde_json")]
1027+
#[bincode(with = "turbo_bincode::serde_self_describing")]
10281028
url_imports: Option<serde_json::Value>,
10291029
/// This option is to enable running the Webpack build in a worker thread
10301030
/// (doesn't apply to Turbopack).
@@ -1397,7 +1397,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>);
13971397

13981398
#[turbo_tasks::value(transparent)]
13991399
pub struct SwcPlugins(
1400-
#[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>,
1400+
#[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>,
14011401
);
14021402

14031403
#[turbo_tasks::value(transparent)]
@@ -1415,7 +1415,7 @@ pub struct OptionServerActions(Option<ServerActions>);
14151415

14161416
#[turbo_tasks::value(transparent)]
14171417
pub struct OptionJsonValue(
1418-
#[bincode(with = "turbo_bincode::serde_json")] pub Option<serde_json::Value>,
1418+
#[bincode(with = "turbo_bincode::serde_self_describing")] pub Option<serde_json::Value>,
14191419
);
14201420

14211421
fn turbopack_config_documentation_link() -> RcStr {
@@ -1468,7 +1468,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue {
14681468

14691469
#[turbo_tasks::value(shared)]
14701470
struct InvalidLoaderRuleConditionIssue {
1471-
#[bincode(with = "turbo_bincode::serde_json")]
1471+
#[bincode(with = "turbo_bincode::serde_self_describing")]
14721472
condition: ConfigConditionItem,
14731473
config_file_path: FileSystemPath,
14741474
}
@@ -2218,7 +2218,7 @@ impl NextConfig {
22182218
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
22192219
#[serde(rename_all = "camelCase")]
22202220
pub struct JsConfig {
2221-
#[bincode(with = "turbo_bincode::serde_json")]
2221+
#[bincode(with = "turbo_bincode::serde_self_describing")]
22222222
compiler_options: Option<serde_json::Value>,
22232223
}
22242224

‎turbopack/crates/turbo-bincode/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,5 @@ indexmap = { workspace = true }
1717
mime = { workspace = true }
1818
ringmap = { workspace = true }
1919
serde = { workspace = true }
20-
serde_json = { workspace = true }
2120
smallvec = { workspace = true }
2221
unty = { workspace = true }

‎turbopack/crates/turbo-bincode/src/lib.rs‎

Lines changed: 1 addition & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#[doc(hidden)]
22
pub mod macro_helpers;
3+
pub mod serde_self_describing;
34

45
use std::{any::Any, ptr::copy_nonoverlapping};
56

@@ -472,76 +473,6 @@ pub mod mime_option {
472473
}
473474
}
474475

475-
/// Encode/decode as a serialized string encoded using `serde_json`.
476-
///
477-
/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known
478-
/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're
479-
/// unsure if the underlying type may trigger a serde compatibility issue.
480-
///
481-
/// In the future this could be replaced with a more efficient serde-compatible self-describing
482-
/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is
483-
/// convenient because it avoids introducing additional dependencies.
484-
///
485-
/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues
486-
pub mod serde_json {
487-
use super::*;
488-
489-
pub fn encode<E: Encoder, T: serde::Serialize>(
490-
value: &T,
491-
encoder: &mut E,
492-
) -> Result<(), EncodeError> {
493-
let json_str =
494-
::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?;
495-
Encode::encode(&json_str, encoder)
496-
}
497-
498-
pub fn decode<Context, D: Decoder<Context = Context>, T: serde::de::DeserializeOwned>(
499-
decoder: &mut D,
500-
) -> Result<T, DecodeError> {
501-
let json_str: String = Decode::decode(decoder)?;
502-
::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
503-
}
504-
505-
pub fn borrow_decode<
506-
'de,
507-
Context,
508-
D: BorrowDecoder<'de, Context = Context>,
509-
T: serde::de::Deserialize<'de>,
510-
>(
511-
decoder: &mut D,
512-
) -> Result<T, DecodeError> {
513-
let json_str: &str = BorrowDecode::borrow_decode(decoder)?;
514-
::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
515-
}
516-
517-
#[cfg(test)]
518-
mod tests {
519-
use ::serde_json::{Value, json};
520-
use bincode::{decode_from_slice, encode_to_vec};
521-
522-
use super::*;
523-
524-
#[test]
525-
fn test_roundtrip() {
526-
let cfg = bincode::config::standard();
527-
528-
#[derive(Encode, Decode)]
529-
struct Wrapper(#[bincode(with = "crate::serde_json")] Value);
530-
531-
let value1 = Wrapper(json!({
532-
"key1": [1, 2, 3],
533-
"key2": [4, 5, 6]
534-
}));
535-
536-
let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg)
537-
.unwrap()
538-
.0;
539-
540-
assert_eq!(value1.0, value2.0);
541-
}
542-
}
543-
}
544-
545476
pub mod either {
546477
use ::either::Either;
547478

0 commit comments

Comments
 (0)