Skip to content

Commit 8018a0d

Browse files
bosconiclaude
andcommitted
environmentd: keep a restarted read-only process from booting writable
Address review on the in-place restart. The entrypoint now exports `MZ_RESTARTED_IN_PLACE=true` before it restarts environmentd, and `preflight_0dt` halts instead of booting writable when that is set and the catalog is at the process's own generation. A process restarts in place only while it is read-only and has not announced itself ready to promote, so that combination means another process of its generation, in a pod that replaced this one, has been promoted. Booting writable would fence it. Without the marker, a stale process could restart while the replacement was being promoted and fence it once. With that guard, the storage controller's read-only halt for a collection deleted during bootstrap uses `restart_in_place!` too, and keeps `halt!` when writable. `restart_in_place!` exits with 75 (`EX_TEMPFAIL`) instead of 168, which is also what a shell reports for a process killed by signal 40. Restore `tini -g` and the entrypoint's TERM/INT traps, so environmentd receives SIGTERM itself and the shell exits instead of restarting or sleeping after it. Shorten the halt in `check_ddl_changes` once the generation has been promoted to "deployment generation N has been promoted; halting". Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 622efbc commit 8018a0d

10 files changed

Lines changed: 68 additions & 27 deletions

File tree

‎misc/python/materialize/cli/run.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -424,8 +424,8 @@ def _kill_childpg():
424424
while True:
425425
last_start_time = datetime.now(UTC)
426426
proc = subprocess.run(command, env=env)
427-
# 166: `halt!`, 168: `restart_in_place!`.
428-
if proc.returncode in (166, 168):
427+
# 166: `halt!`, 75: `restart_in_place!`.
428+
if proc.returncode in (166, 75):
429429
wait = max(
430430
timedelta(seconds=5) - (datetime.now(UTC) - last_start_time),
431431
timedelta(seconds=0),

‎src/environmentd/ci/Dockerfile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ MZFROM distroless-prod-base
1414
COPY environmentd entrypoint.sh /usr/local/bin/
1515
COPY --from=openssh /output/ssh /usr/bin/ssh
1616

17-
# tini (from distroless-prod-base) is PID 1 so ssh tunnel subprocesses get
18-
# reaped. entrypoint.sh runs environmentd, then sleeps on graceful exit. Run it
19-
# through busybox `sh` explicitly, as distroless has no /bin/sh.
20-
ENTRYPOINT ["/usr/bin/tini", "--", "/busybox/sh", "/usr/local/bin/entrypoint.sh"]
17+
# tini reaps orphaned subprocesses and forwards signals to the whole process
18+
# group, including environmentd. Run entrypoint.sh through busybox `sh`
19+
# explicitly, as distroless has no /bin/sh.
20+
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/busybox/sh", "/usr/local/bin/entrypoint.sh"]

‎src/environmentd/ci/entrypoint.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,12 @@ fi
2121
# halts that are safe to repeat inside this container. Restarting here avoids
2222
# Kubernetes crash backoff. Every other failure, including mz_ore::halt!
2323
# (166), exits the container and leaves the restart to Kubernetes.
24-
restart_in_place=168
24+
restart_in_place=75
25+
26+
# tini -g forwards shutdown signals to both this shell and environmentd.
27+
# Exit even if environmentd handles the signal by halting or exiting cleanly.
28+
trap 'exit 143' TERM
29+
trap 'exit 130' INT
2530

2631
while true; do
2732
if environmentd "$@"; then
@@ -33,5 +38,7 @@ while true; do
3338
exit "$status"
3439
fi
3540
echo "environmentd requested a restart in place; restarting" >&2
41+
# Keeps the restarted process from booting writable, see preflight_0dt.
42+
export MZ_RESTARTED_IN_PLACE=true
3643
fi
3744
done

‎src/environmentd/src/deployment/preflight.rs‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,13 @@ pub struct CatchupConfig {
4848
}
4949

5050
/// Returns whether this deployment should boot in read-only mode.
51+
///
52+
/// `restarted_in_place` is whether the container entrypoint restarted this
53+
/// process after [`restart_in_place!`].
5154
pub async fn preflight_0dt(
5255
openable_adapter_storage: &mut dyn OpenableDurableCatalogState,
5356
deploy_generation: u64,
57+
restarted_in_place: bool,
5458
) -> Result<bool, CatalogError> {
5559
if !openable_adapter_storage.is_initialized().await? {
5660
info!("catalog not initialized; booting with writes allowed");
@@ -63,6 +67,18 @@ pub async fn preflight_0dt(
6367
info!("this deployment is a new generation; booting in read only mode");
6468
Ok(true)
6569
} else if catalog_generation == deploy_generation {
70+
// A process restarts in place only while it is read-only and has not
71+
// announced itself ready to promote, so finding its own generation in
72+
// the catalog means another process of this generation, in a pod that
73+
// replaced this one, has been promoted. Booting writable would fence
74+
// that process. Kubernetes does not restart a deleted pod, so leave the
75+
// restart to it.
76+
if restarted_in_place {
77+
halt!(
78+
"restarted in place, but deployment generation {} has been promoted; halting",
79+
catalog_generation
80+
);
81+
}
6682
info!("this deployment is the current generation; booting with writes allowed");
6783
Ok(false)
6884
} else {
@@ -340,12 +356,9 @@ async fn check_ddl_changes(
340356

341357
let extra_info = separated(". ", info_parts);
342358

343-
// A restart in place boots a new process even if Kubernetes has replaced
344-
// this pod in the meantime. That is safe while this generation is still
345-
// read-only: the new process opens the catalog in savepoint mode again and
346-
// does not fence anyone. Once a process of this generation has been
347-
// promoted, a new process would boot as the leader and fence the live one,
348-
// so leave the restart to Kubernetes.
359+
// Restart in place only while this generation is still read-only. Once a
360+
// process of this generation has been promoted, a restarted process would
361+
// only halt in `preflight_0dt`, so leave the restart to Kubernetes.
349362
let catalog_generation = openable_catalog(
350363
persist_client,
351364
&environment_id,
@@ -363,10 +376,8 @@ async fn check_ddl_changes(
363376
)
364377
} else {
365378
halt!(
366-
"there have been DDL that we need to react to, and generation {} has been \
367-
promoted; halting. {}",
368-
catalog_generation,
369-
extra_info
379+
"deployment generation {} has been promoted; halting",
380+
catalog_generation
370381
)
371382
}
372383
}
@@ -468,7 +479,7 @@ mod tests {
468479
async fn catchup_starts_after_bootstrap() {
469480
let (builder, config, handle) = setup().await;
470481
let mut openable = builder.with_deploy_generation(1).unwrap_build().await;
471-
assert!(preflight_0dt(openable.as_mut(), 1).await.unwrap());
482+
assert!(preflight_0dt(openable.as_mut(), 1, false).await.unwrap());
472483
let (_trigger, receiver) = trigger::channel();
473484
let (bootstrapped, bootstrapped_receiver) = oneshot::channel();
474485
let metrics = Arc::clone(&config.catalog_metrics);

‎src/environmentd/src/environmentd/main.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ pub struct Args {
104104
/// this should never be enabled in CI.
105105
#[clap(long, env = "ALL_FEATURES")]
106106
all_features: bool,
107+
/// Set by the container entrypoint when it restarts environmentd after
108+
/// `mz_ore::restart_in_place!`.
109+
#[clap(long, env = "RESTARTED_IN_PLACE", hide = true)]
110+
restarted_in_place: bool,
107111

108112
// === Connection options. ===
109113
/// Path to a file containing the json-formatted configuration of our
@@ -1110,6 +1114,7 @@ fn run(mut args: Args) -> Result<(), anyhow::Error> {
11101114
// Special modes.
11111115
unsafe_mode: args.unsafe_mode,
11121116
all_features: args.all_features,
1117+
restarted_in_place: args.restarted_in_place,
11131118
// Connection options.
11141119
tls,
11151120
tls_reload_certs: mz_server_core::default_cert_reload_ticker(),

‎src/environmentd/src/lib.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ pub struct Config {
100100
/// Whether the environmentd is running on a local dev machine. This is
101101
/// never meant to run in production or CI.
102102
pub all_features: bool,
103+
/// Whether the container entrypoint restarted this process after
104+
/// `mz_ore::restart_in_place!`.
105+
pub restarted_in_place: bool,
103106

104107
// === Connection options. ===
105108
/// TLS encryption and authentication configuration.
@@ -671,6 +674,7 @@ impl Listeners {
671674
let read_only = preflight::preflight_0dt(
672675
openable_adapter_storage.as_mut(),
673676
config.controller.deploy_generation,
677+
config.restarted_in_place,
674678
)
675679
.await?;
676680

‎src/environmentd/src/test_util.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -932,6 +932,7 @@ impl Listeners {
932932
frontegg_oauth_issuer_url: None,
933933
unsafe_mode: config.unsafe_mode,
934934
all_features: false,
935+
restarted_in_place: false,
935936
metrics_registry: metrics_registry.clone(),
936937
now: config.now,
937938
environment_id: config.environment_id,

‎src/ore/src/process.rs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,11 @@
1919
pub const HALT_EXIT_CODE: i32 = 166;
2020

2121
/// The exit code of [`restart_in_place!`](crate::restart_in_place).
22-
pub const RESTART_IN_PLACE_EXIT_CODE: i32 = 168;
22+
///
23+
/// This is `EX_TEMPFAIL` from `sysexits.h`. Shells, tini and Kubernetes report
24+
/// a process killed by signal N as exit code 128 + N, so a code below 128 can
25+
/// only come from an explicit exit.
26+
pub const RESTART_IN_PLACE_EXIT_CODE: i32 = 75;
2327

2428
/// Halts the process.
2529
///
@@ -61,10 +65,11 @@ macro_rules! halt {
6165
/// halts repeatedly.
6266
///
6367
/// The restarted process runs even if Kubernetes has replaced the process's pod
64-
/// in the meantime, for example after a node failure. Only use this macro where
65-
/// booting a second process next to the replacement cannot take over from it,
66-
/// e.g., a deployment that reboots in read-only mode. Use [`halt!`](crate::halt)
67-
/// everywhere else, in particular after the process has been fenced out.
68+
/// in the meantime, for example after a node failure. Only use this macro while
69+
/// the deployment is read-only, where the entrypoint's restart marker keeps the
70+
/// restarted process from booting writable next to the replacement. Use
71+
/// [`halt!`](crate::halt) everywhere else, in particular after the process has
72+
/// been fenced out.
6873
#[cfg_attr(nightly_doc_features, doc(cfg(feature = "tracing")))]
6974
#[cfg(feature = "tracing")]
7075
#[macro_export]

‎src/sqllogictest/src/runner.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,7 @@ impl<'a> RunnerInner<'a> {
13421342
cors_allowed_origin_list: Vec::new(),
13431343
unsafe_mode: true,
13441344
all_features: false,
1345+
restarted_in_place: false,
13451346
metrics_registry,
13461347
now,
13471348
environment_id,

‎src/storage-controller/src/lib.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use mz_ore::collections::CollectionExt;
4040
use mz_ore::metrics::MetricsRegistry;
4141
use mz_ore::now::NowFn;
4242
use mz_ore::task::AbortOnDropHandle;
43-
use mz_ore::{assert_none, halt, instrument, soft_panic_or_log};
43+
use mz_ore::{assert_none, halt, instrument, restart_in_place, soft_panic_or_log};
4444
use mz_persist_client::batch::ProtoBatch;
4545
use mz_persist_client::cache::PersistClientCache;
4646
use mz_persist_client::cfg::USE_CRITICAL_SINCE_SNAPSHOT;
@@ -1028,13 +1028,20 @@ impl StorageController for Controller {
10281028
// halting on it would wedge bootstrap forever instead of
10291029
// resolving on the next attempt.
10301030
if dependency_since.is_empty() && !write_frontier.is_empty() {
1031-
halt!(
1031+
let reason = format!(
10321032
"dependency since frontier is empty while dependent upper \
10331033
is not empty (dependent id={id}, write_frontier={:?}, dependency_read_holds={:?}), \
10341034
this indicates concurrent deletion of a collection",
1035-
write_frontier,
1036-
dependency_read_holds,
1035+
write_frontier, dependency_read_holds,
10371036
);
1037+
// A read-only controller belongs to a deployment that is
1038+
// still catching up, which can restart without Kubernetes
1039+
// crash backoff.
1040+
if self.read_only {
1041+
restart_in_place!("{}", reason);
1042+
} else {
1043+
halt!("{}", reason);
1044+
}
10381045
}
10391046

10401047
// If the dependency is between a remap shard and a source (and

0 commit comments

Comments
 (0)