Skip to content

Commit 64bb8c9

Browse files
steipetesteipete
committed
fix: back up live data before FakeCo updates
1 parent 184e6d7 commit 64bb8c9

3 files changed

Lines changed: 333 additions & 37 deletions

File tree

‎deploy/fakeco/aws/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,19 @@ workflow's expected SHA-256 before invoking `bash`, so a swapped object never
250250
executes. Its required bucket listing is limited by `s3:prefix` to
251251
`<artifact-prefix>/owner/*`; it cannot list unrelated bucket keys. The instance
252252
profile cannot fetch GitHub, Parameter Store, or secrets.
253+
254+
Before an update bootstrap installs source, rewrites runtime configuration, or
255+
starts the requested image, it detects existing runtime state and verifies the
256+
one running release and image. For every verified existing runtime, including a
257+
same-commit retry, it probes the current service, takes an integrity-checked hot
258+
SQLite backup, uploads the encrypted database and metadata-only pre-update
259+
evidence under a distinct object ID, creates both objects only when their keys
260+
do not already exist, and verifies both S3 objects. A pristine first apply skips
261+
this step; collisions and partial or unverifiable existing state fail before new
262+
code can start. This small-instance owner accepts backup objects up to
263+
5,000,000,000 bytes; larger databases fail closed before upload or update and
264+
need a separately designed multipart backup path.
265+
253266
Bootstrap:
254267

255268
1. Installs Docker, Compose, SQLite, and probe tools from Noble, then installs

‎deploy/fakeco/aws/bootstrap.sh‎

Lines changed: 177 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,15 @@ backup_runtime_override="$state_root/compose.backup.yaml"
5757
run_id="$(date -u +%Y%m%dT%H%M%SZ)-${CLICKCLACK_SOURCE_COMMIT:0:12}"
5858
log_file="$log_root/$run_id.log"
5959
stage=initialize
60+
pre_update_backup=false
61+
captured_backup_path=
62+
captured_backup_sha=
63+
captured_backup_size=
64+
captured_backup_key=
65+
captured_backup_head=
6066
readonly aws_cli_version=2.35.20
6167
readonly aws_cli_archive_sha256=58799ce9276d4e8815fd19e4dc35649626c6b4fbd4d0e3df7433af9cfde41882
68+
readonly max_single_put_bytes=5000000000
6269

6370
install -d -m 0750 "$owner_root" "$release_root" "$state_root" "$log_root" "$(dirname "$runtime_env")"
6471
touch "$log_file"
@@ -131,7 +138,7 @@ verify_persistent_runtime_config() {
131138
[[ -f "$runtime_env" && -f "$runtime_override" ]] || return 1
132139
configured_images="$(grep -c '^ image:' "$runtime_override" || true)"
133140
[[ "$configured_images" == "2" ]] || return 1
134-
if [[ "$action" == "backup" ]]; then
141+
if [[ "$action" == "backup" || "$pre_update_backup" == "true" ]]; then
135142
grep -Eq "^CLICKCLACK_WEB_VERSION=($requested_source_commit|$runtime_source_commit)$" "$runtime_env" || return 1
136143
requested_images="$(grep -Fxc " image: \"clickclack:fakeco-$requested_source_commit\"" "$runtime_override" || true)"
137144
if [[ "$requested_source_commit" == "$runtime_source_commit" ]]; then
@@ -407,44 +414,142 @@ cleanup_run_files() {
407414
fi
408415
}
409416

410-
create_backup() {
411-
stage=sqlite-backup
412-
local container_path mount_path host_path integrity backup_sha backup_size backup_key backup_head manifest_key log_key
413-
container_path="/app/data/backups/clickclack-$run_id.db"
417+
capture_backup() {
418+
local backup_id="$1"
419+
local stage_prefix="$2"
420+
local container_path mount_path integrity
421+
[[ "$backup_id" =~ ^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$ ]]
422+
[[ -z "$stage_prefix" || "$stage_prefix" == "pre-update-" ]]
423+
stage="${stage_prefix}sqlite-backup"
424+
container_path="/app/data/backups/clickclack-$backup_id.db"
414425
compose exec -T app sh -c 'mkdir -p /app/data/backups'
415426
compose exec -T app clickclack backup --data /app/data --out "$container_path"
416427
mount_path="$(docker volume inspect clickclack-fakeco-data --format '{{.Mountpoint}}')"
417-
host_path="$mount_path/backups/clickclack-$run_id.db"
418-
[[ -f "$host_path" ]]
419-
integrity="$(sqlite3 "$host_path" 'PRAGMA integrity_check;')"
428+
captured_backup_path="$mount_path/backups/clickclack-$backup_id.db"
429+
[[ -f "$captured_backup_path" ]]
430+
integrity="$(sqlite3 "$captured_backup_path" 'PRAGMA integrity_check;')"
420431
[[ "$integrity" == "ok" ]]
421-
backup_sha="$(sha256sum "$host_path" | cut -d' ' -f1)"
422-
backup_size="$(stat -c '%s' "$host_path")"
423-
backup_key="$CLICKCLACK_BACKUP_PREFIX/sqlite/$runtime_source_commit/clickclack-$run_id.db"
424-
backup_head="$state_root/backup-head-$run_id.json"
425-
manifest_key="$CLICKCLACK_BACKUP_PREFIX/manifests/$run_id.json"
426-
log_key="$CLICKCLACK_LOG_PREFIX/runs/$run_id/owner.log"
432+
captured_backup_sha="$(sha256sum "$captured_backup_path" | cut -d' ' -f1)"
433+
captured_backup_size="$(stat -c '%s' "$captured_backup_path")"
434+
[[ "$captured_backup_size" =~ ^[0-9]+$ ]]
435+
if ((captured_backup_size > max_single_put_bytes)); then
436+
printf 'backup size exceeds the FakeCo single-object limit (%s bytes)\n' \
437+
"$max_single_put_bytes" >&2
438+
return 1
439+
fi
440+
captured_backup_key="$CLICKCLACK_BACKUP_PREFIX/sqlite/$runtime_source_commit/clickclack-$backup_id.db"
441+
captured_backup_head="$state_root/backup-head-$backup_id.json"
427442

428-
stage=upload-backup
429-
aws s3 cp "$host_path" "s3://$CLICKCLACK_BACKUP_BUCKET/$backup_key" \
430-
--only-show-errors \
431-
--metadata "sha256=$backup_sha,source-commit=$runtime_source_commit" \
432-
--sse aws:kms \
433-
--sse-kms-key-id "$CLICKCLACK_DATA_KMS_KEY_ARN"
443+
stage="${stage_prefix}upload-backup"
444+
aws s3api put-object \
445+
--bucket "$CLICKCLACK_BACKUP_BUCKET" \
446+
--key "$captured_backup_key" \
447+
--body "$captured_backup_path" \
448+
--content-type application/vnd.sqlite3 \
449+
--metadata "sha256=$captured_backup_sha,source-commit=$runtime_source_commit" \
450+
--server-side-encryption aws:kms \
451+
--ssekms-key-id "$CLICKCLACK_DATA_KMS_KEY_ARN" \
452+
--if-none-match '*' \
453+
--output json >/dev/null
434454
aws s3api head-object \
435455
--bucket "$CLICKCLACK_BACKUP_BUCKET" \
436-
--key "$backup_key" \
437-
--output json >"$backup_head"
456+
--key "$captured_backup_key" \
457+
--output json >"$captured_backup_head"
438458
jq -e \
439-
--arg sha256 "$backup_sha" \
459+
--arg sha256 "$captured_backup_sha" \
440460
--arg source_commit "$runtime_source_commit" \
441461
--arg kms_key "$CLICKCLACK_DATA_KMS_KEY_ARN" \
442-
--argjson size "$backup_size" \
462+
--argjson size "$captured_backup_size" \
443463
'.Metadata.sha256 == $sha256 and
444464
.Metadata["source-commit"] == $source_commit and
445465
.ServerSideEncryption == "aws:kms" and
446466
.SSEKMSKeyId == $kms_key and
447-
.ContentLength == $size' "$backup_head" >/dev/null
467+
.ContentLength == $size' "$captured_backup_head" >/dev/null
468+
}
469+
470+
create_pre_update_backup() {
471+
local backup_id backup_suffix manifest_key evidence_file evidence_sha evidence_size evidence_head
472+
backup_suffix="$(printf 'pre-update:%s:%s:%s' \
473+
"$run_id" "$runtime_source_commit" "$CLICKCLACK_OWNER_COMMIT" | sha256sum | cut -c1-12)"
474+
backup_id="${run_id%-*}-$backup_suffix"
475+
[[ "$backup_id" != "$run_id" ]]
476+
capture_backup "$backup_id" pre-update-
477+
manifest_key="$CLICKCLACK_BACKUP_PREFIX/manifests/$backup_id.json"
478+
evidence_file="$state_root/pre-update-evidence-$run_id.json"
479+
evidence_head="$state_root/pre-update-evidence-head-$run_id.json"
480+
jq -n \
481+
--arg action pre-update \
482+
--arg run_id "$backup_id" \
483+
--arg source_commit "$runtime_source_commit" \
484+
--arg requested_source_commit "$requested_source_commit" \
485+
--arg owner_commit "$CLICKCLACK_OWNER_COMMIT" \
486+
--arg image_id "$image_id" \
487+
--arg backup_bucket "$CLICKCLACK_BACKUP_BUCKET" \
488+
--arg backup_key "$captured_backup_key" \
489+
--arg backup_sha256 "$captured_backup_sha" \
490+
--argjson backup_size "$captured_backup_size" \
491+
--arg manifest_bucket "$CLICKCLACK_BACKUP_BUCKET" \
492+
--arg manifest_key "$manifest_key" \
493+
'{
494+
schema_version: 1,
495+
status: "passed",
496+
action: $action,
497+
run_id: $run_id,
498+
source_commit: $source_commit,
499+
requested_source_commit: $requested_source_commit,
500+
owner_commit: $owner_commit,
501+
runtime_commit_verified: true,
502+
image_id: $image_id,
503+
health: true,
504+
readiness: true,
505+
metrics_metadata_only: true,
506+
integrity_check: "ok",
507+
backup: {
508+
bucket: $backup_bucket,
509+
key: $backup_key,
510+
sha256: $backup_sha256,
511+
size_bytes: $backup_size
512+
},
513+
manifest: {bucket: $manifest_bucket, key: $manifest_key}
514+
}' >"$evidence_file"
515+
evidence_sha="$(sha256sum "$evidence_file" | cut -d' ' -f1)"
516+
evidence_size="$(stat -c '%s' "$evidence_file")"
517+
518+
stage=pre-update-upload-evidence
519+
aws s3api put-object \
520+
--bucket "$CLICKCLACK_BACKUP_BUCKET" \
521+
--key "$manifest_key" \
522+
--body "$evidence_file" \
523+
--content-type application/json \
524+
--metadata "sha256=$evidence_sha,source-commit=$runtime_source_commit,requested-source-commit=$requested_source_commit" \
525+
--server-side-encryption aws:kms \
526+
--ssekms-key-id "$CLICKCLACK_DATA_KMS_KEY_ARN" \
527+
--if-none-match '*' \
528+
--output json >/dev/null
529+
aws s3api head-object \
530+
--bucket "$CLICKCLACK_BACKUP_BUCKET" \
531+
--key "$manifest_key" \
532+
--output json >"$evidence_head"
533+
jq -e \
534+
--arg sha256 "$evidence_sha" \
535+
--arg source_commit "$runtime_source_commit" \
536+
--arg requested_source_commit "$requested_source_commit" \
537+
--arg kms_key "$CLICKCLACK_DATA_KMS_KEY_ARN" \
538+
--argjson size "$evidence_size" \
539+
'.Metadata.sha256 == $sha256 and
540+
.Metadata["source-commit"] == $source_commit and
541+
.Metadata["requested-source-commit"] == $requested_source_commit and
542+
.ServerSideEncryption == "aws:kms" and
543+
.SSEKMSKeyId == $kms_key and
544+
.ContentLength == $size' "$evidence_head" >/dev/null
545+
rm -f -- "$captured_backup_path" "$captured_backup_head" "$evidence_file" "$evidence_head"
546+
}
547+
548+
create_backup() {
549+
local manifest_key log_key
550+
capture_backup "$run_id" ""
551+
manifest_key="$CLICKCLACK_BACKUP_PREFIX/manifests/$run_id.json"
552+
log_key="$CLICKCLACK_LOG_PREFIX/runs/$run_id/owner.log"
448553
compose logs --no-color --tail 500 app >"$state_root/app-$run_id.log"
449554

450555
evidence_file="$state_root/evidence-$run_id.json"
@@ -457,8 +562,8 @@ create_backup() {
457562
--arg image_id "$image_id" \
458563
--arg seed_sha256 "$seed_sha256" \
459564
--arg backup_bucket "$CLICKCLACK_BACKUP_BUCKET" \
460-
--arg backup_key "$backup_key" \
461-
--arg backup_sha256 "$backup_sha" \
565+
--arg backup_key "$captured_backup_key" \
566+
--arg backup_sha256 "$captured_backup_sha" \
462567
--arg manifest_bucket "$CLICKCLACK_BACKUP_BUCKET" \
463568
--arg manifest_key "$manifest_key" \
464569
--arg log_bucket "$CLICKCLACK_LOG_BUCKET" \
@@ -485,12 +590,16 @@ create_backup() {
485590
}' >"$evidence_file"
486591

487592
stage=upload-evidence
488-
aws s3 cp "$evidence_file" "s3://$CLICKCLACK_BACKUP_BUCKET/$manifest_key" \
489-
--only-show-errors \
593+
aws s3api put-object \
594+
--bucket "$CLICKCLACK_BACKUP_BUCKET" \
595+
--key "$manifest_key" \
596+
--body "$evidence_file" \
490597
--content-type application/json \
491-
--sse aws:kms \
492-
--sse-kms-key-id "$CLICKCLACK_DATA_KMS_KEY_ARN"
493-
cleanup_success "$host_path"
598+
--server-side-encryption aws:kms \
599+
--ssekms-key-id "$CLICKCLACK_DATA_KMS_KEY_ARN" \
600+
--if-none-match '*' \
601+
--output json >/dev/null
602+
cleanup_success "$captured_backup_path"
494603
stage=upload-log
495604
aws s3 cp "$log_file" "s3://$CLICKCLACK_LOG_BUCKET/$log_key" \
496605
--only-show-errors \
@@ -501,7 +610,43 @@ create_backup() {
501610
cat "$evidence_file" >&3
502611
}
503612

613+
runtime_state_exists() {
614+
local containers
615+
if [[ -e "$runtime_env" || -e "$runtime_override" || -e /var/lib/docker/volumes/clickclack-fakeco-data ]]; then
616+
return 0
617+
fi
618+
if command -v docker >/dev/null && systemctl is-active --quiet docker; then
619+
containers="$(docker ps -a \
620+
--filter 'label=com.docker.compose.project=clickclack-fakeco' \
621+
--filter 'label=com.docker.compose.service=app' \
622+
--format '{{.ID}}')"
623+
[[ -n "$containers" ]] && return 0
624+
fi
625+
return 1
626+
}
627+
628+
prepare_pre_update_backup() {
629+
stage=pre-update-detection
630+
runtime_state_exists || return 0
631+
[[ -f "$runtime_env" && ! -L "$runtime_env" ]]
632+
[[ -f "$runtime_override" && ! -L "$runtime_override" ]]
633+
systemctl is-active --quiet docker
634+
pre_update_backup=true
635+
resolve_backup_runtime
636+
verify_running_image
637+
probe_service
638+
create_pre_update_backup
639+
rm -f -- "$backup_runtime_override"
640+
pre_update_backup=false
641+
runtime_source_commit="$requested_source_commit"
642+
runtime_source_sha256="$CLICKCLACK_SOURCE_SHA256"
643+
compose_override="$runtime_override"
644+
set_runtime_paths
645+
stage=pre-update-complete
646+
}
647+
504648
if [[ "$action" == "bootstrap" ]]; then
649+
prepare_pre_update_backup
505650
install_runtime
506651
install_source
507652
write_runtime_config

0 commit comments

Comments
 (0)