Skip to content

Commit a34a0d0

Browse files
authored
runner: accept a pre-integrity ledger of the current layout on open (#631)
Opening a ledger written before the integrity columns, in the current table layout, no longer fails: the store accepts it and the native-live report and store tests pass again. Merge gate: pineforge/verify (clean-tree cloud jobs) and pineforge/parity (full sweep, no coverage lost, no hard regression, no leavers) on this head.
1 parent 11eae42 commit a34a0d0

3 files changed

Lines changed: 177 additions & 2 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ unchanged except for two additive diagnostics keys, described below.
8484
earlier runners cannot open the migrated database. The summaries add one
8585
row and up to three partial-index entries per action, plus per-target rows
8686
and indexes.
87+
- The ledger open derives the table layouts it accepts from one list of the
88+
ledger's tables: every layout a runner has written, each also without
89+
`report_integrity`; any other table count is refused as before. A ledger
90+
without integrity digests, including one that already has the delivery
91+
summaries, opens and needs one verified `run` replay before export.
8792

8893
### Run profile in the report diagnostics
8994

‎runner/store.cpp‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <cerrno>
1111
#include <chrono>
1212
#include <filesystem>
13+
#include <iterator>
1314
#include <limits>
1415
#include <mutex>
1516
#include <new>
@@ -386,6 +387,36 @@ void migrate_delivery_state(sqlite3* db) {
386387
"THEN last_error_log ELSE MAX(last_error_log,excluded.last_error_log) END; END;");
387388
}
388389

390+
// Every table the ledger open creates, by the layout stage that added it. The
391+
// tables up to the end of a stage form a layout some runner wrote; stages 3
392+
// and 4 are the layouts written before redelivery requests and before
393+
// integrity digests.
394+
struct LedgerTable {
395+
const char* name;
396+
unsigned stage;
397+
};
398+
constexpr LedgerTable ledger_tables[] = {
399+
{"metadata", 1}, {"inputs", 1}, {"events", 1},
400+
{"routing_configuration", 2}, {"event_routes", 2}, {"delivery_log", 2},
401+
{"report_snapshots", 3}, {"redelivery_requests", 4}, {"report_integrity", 5},
402+
{"delivery_totals", 6}, {"delivery_targets", 6}, {"delivery_state", 6},
403+
};
404+
405+
// Admit every written layout, and each one without report_integrity: the open
406+
// creates the missing tables and one verified replay populates the digests.
407+
bool supported_layout(std::uint64_t table_count) {
408+
std::uint64_t tables = 0;
409+
bool integrity = false;
410+
for (std::size_t i = 0; i < std::size(ledger_tables); ++i) {
411+
++tables;
412+
integrity = integrity || std::string_view(ledger_tables[i].name) == "report_integrity";
413+
if (i + 1 < std::size(ledger_tables) && ledger_tables[i + 1].stage == ledger_tables[i].stage)
414+
continue;
415+
if (table_count == tables || (integrity && table_count == tables - 1)) return true;
416+
}
417+
return false;
418+
}
419+
389420
} // namespace
390421

391422
LedgerBudgetExceeded::LedgerBudgetExceeded(std::uint64_t used_bytes, std::uint64_t requested_bytes,
@@ -615,8 +646,7 @@ Ledger::Ledger(const std::string& path, const std::string& deployment_identity,
615646
Statement q(impl_->db, "INSERT INTO metadata VALUES(1,1,?)");
616647
q.bind(1, deployment_identity);
617648
q.done();
618-
} else if (table_count != 3 && table_count != 6 && table_count != 7 && table_count != 8 && table_count != 9 &&
619-
table_count != 12) {
649+
} else if (!supported_layout(table_count)) {
620650
throw std::runtime_error("native ledger is not an empty or supported ledger database");
621651
}
622652
{

‎tests/test_native_live_store.cpp‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -863,6 +863,144 @@ void cumulative_reports() {
863863
CHECK(LedgerView(path).report_json() == "{\"input_cursor\":1}");
864864
}
865865

866+
std::string table_count(const std::string& path) {
867+
return scalar_text(path, "SELECT COUNT(*) FROM sqlite_master WHERE type='table'");
868+
}
869+
870+
// The open's refusal text, or empty when the ledger opens.
871+
std::string open_refusal(const std::string& path, const std::string& identity) {
872+
try { Ledger ledger(path, identity); }
873+
catch (const std::exception& error) { return error.what(); }
874+
return "";
875+
}
876+
877+
// A ledger written before integrity digests, prepared as the report end-to-end
878+
// test does: the current layout without report_integrity. It opens, export
879+
// refuses until one verified replay populates the digests, and the replay
880+
// restores the digests and the export the ledger had.
881+
void pre_integrity_ledger() {
882+
TempDir temp;
883+
const auto path = temp.file("pre-integrity.sqlite");
884+
const std::vector<std::string> reports{"{\"input_cursor\":0}", "{\"input_cursor\":1}", "{\"input_cursor\":2}"};
885+
{
886+
Ledger ledger(path, "pre-integrity");
887+
ledger.verify_report(0, reports[0]);
888+
ledger.commit_input(0, "{}", 1, {{"first", "{}"}}, reports[1]);
889+
ledger.commit_input(1, "{}", 2, {}, reports[2]);
890+
}
891+
const auto exported = LedgerView(path).report_json();
892+
const auto digests = source_rows(path, "SELECT input_cursor,digest FROM report_integrity ORDER BY input_cursor");
893+
CHECK(digests.size() == reports.size());
894+
execute_sql(path, "DROP TABLE report_integrity;");
895+
CHECK(table_count(path) == "11");
896+
CHECK(throws([&] { LedgerView(path).report_json(); }));
897+
std::unique_ptr<Ledger> resumed;
898+
try { resumed = std::make_unique<Ledger>(path, "pre-integrity"); }
899+
catch (const std::exception& error) { std::fprintf(stderr, "11-table ledger refused: %s\n", error.what()); }
900+
CHECK(resumed);
901+
if (!resumed) return;
902+
CHECK(table_count(path) == "12");
903+
CHECK(throws([&] { LedgerView(path).report_json(); }));
904+
for (std::uint64_t cursor = 0; cursor < reports.size(); ++cursor)
905+
resumed->verify_report(cursor, reports[cursor]);
906+
CHECK(source_rows(path, "SELECT input_cursor,digest FROM report_integrity ORDER BY input_cursor") == digests);
907+
CHECK(LedgerView(path).report_json() == exported);
908+
resumed.reset();
909+
CHECK(open_refusal(path, "pre-integrity").empty());
910+
}
911+
912+
// Each table layout a runner wrote, built with that runner's own statements,
913+
// opens and leaves the current layout. Any other table count is refused and
914+
// keeps its tables: a written layout plus unrelated tables would otherwise open.
915+
void ledger_layouts() {
916+
TempDir temp;
917+
const char* const first =
918+
"CREATE TABLE metadata (singleton INTEGER PRIMARY KEY CHECK(singleton=1),"
919+
"schema_version INTEGER NOT NULL CHECK(schema_version=1),identity TEXT NOT NULL);"
920+
"CREATE TABLE inputs (input_index INTEGER PRIMARY KEY CHECK(input_index>=0),"
921+
"canonical_json TEXT NOT NULL,state_hash TEXT NOT NULL);"
922+
"CREATE TABLE events (ordinal INTEGER PRIMARY KEY CHECK(ordinal>0),"
923+
"input_index INTEGER NOT NULL REFERENCES inputs(input_index),"
924+
"input_position INTEGER NOT NULL CHECK(input_position>=0),"
925+
"event_id TEXT NOT NULL UNIQUE,payload TEXT NOT NULL,"
926+
"attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 4294967295),"
927+
"acknowledged INTEGER NOT NULL DEFAULT 0 CHECK(acknowledged IN (0,1)),"
928+
"last_error TEXT NOT NULL DEFAULT '',UNIQUE(input_index,input_position));"
929+
"CREATE INDEX pending_events ON events(ordinal) WHERE acknowledged=0;"
930+
"INSERT INTO metadata VALUES(1,1,'layouts');"
931+
"INSERT INTO inputs VALUES(0,'{}','1');"
932+
"INSERT INTO events(ordinal,input_index,input_position,event_id,payload) VALUES(1,0,0,'first','{}');";
933+
const char* const routing =
934+
"CREATE TABLE IF NOT EXISTS routing_configuration (singleton INTEGER PRIMARY KEY CHECK(singleton=1),document TEXT NOT NULL);"
935+
"CREATE TABLE IF NOT EXISTS event_routes (ordinal INTEGER PRIMARY KEY REFERENCES events(ordinal),"
936+
"target_id TEXT,delivery_id TEXT NOT NULL);"
937+
"CREATE TABLE IF NOT EXISTS delivery_log (log_id INTEGER PRIMARY KEY,event_id TEXT NOT NULL REFERENCES events(event_id),"
938+
"target_id TEXT NOT NULL,delivery_id TEXT NOT NULL,attempt INTEGER NOT NULL CHECK(attempt>0),"
939+
"phase TEXT NOT NULL CHECK(phase IN ('started','completed')),started_at INTEGER NOT NULL,ended_at INTEGER,"
940+
"http_status INTEGER NOT NULL DEFAULT 0,error_category TEXT NOT NULL DEFAULT '',success INTEGER NOT NULL DEFAULT 0 CHECK(success IN (0,1)),"
941+
"UNIQUE(event_id,attempt,phase));"
942+
"CREATE INDEX IF NOT EXISTS delivery_results ON delivery_log(event_id,phase,log_id);"
943+
"CREATE TRIGGER IF NOT EXISTS delivery_log_no_update BEFORE UPDATE ON delivery_log BEGIN SELECT RAISE(ABORT,'delivery log is append-only'); END;"
944+
"CREATE TRIGGER IF NOT EXISTS delivery_log_no_delete BEFORE DELETE ON delivery_log BEGIN SELECT RAISE(ABORT,'delivery log is append-only'); END;"
945+
"INSERT INTO event_routes(ordinal,target_id,delivery_id) SELECT ordinal,'default',event_id FROM events "
946+
"WHERE ordinal NOT IN (SELECT ordinal FROM event_routes);";
947+
const char* const reports =
948+
"CREATE TABLE IF NOT EXISTS report_snapshots (input_cursor INTEGER PRIMARY KEY CHECK(input_cursor>=0),payload TEXT NOT NULL);"
949+
"CREATE TRIGGER IF NOT EXISTS report_no_update BEFORE UPDATE ON report_snapshots BEGIN SELECT RAISE(ABORT,'reports are immutable'); END;"
950+
"CREATE TRIGGER IF NOT EXISTS report_no_delete BEFORE DELETE ON report_snapshots BEGIN SELECT RAISE(ABORT,'reports are immutable'); END;";
951+
const char* const redelivery =
952+
"ALTER TABLE events ADD COLUMN created_at_ms INTEGER NOT NULL DEFAULT 0;"
953+
"ALTER TABLE delivery_log ADD COLUMN request_id TEXT NOT NULL DEFAULT '';"
954+
"CREATE TABLE IF NOT EXISTS redelivery_requests (request_order INTEGER PRIMARY KEY,request_id TEXT NOT NULL UNIQUE,"
955+
"target_id TEXT NOT NULL,from_ordinal INTEGER NOT NULL,through_ordinal INTEGER NOT NULL,"
956+
"failed_only INTEGER NOT NULL CHECK(failed_only IN (0,1)),snapshot_log INTEGER NOT NULL,"
957+
"selected INTEGER NOT NULL,accepted_at INTEGER NOT NULL);"
958+
"CREATE TRIGGER IF NOT EXISTS redelivery_no_update BEFORE UPDATE ON redelivery_requests BEGIN SELECT RAISE(ABORT,'requests are immutable'); END;"
959+
"CREATE TRIGGER IF NOT EXISTS redelivery_no_delete BEFORE DELETE ON redelivery_requests BEGIN SELECT RAISE(ABORT,'requests are immutable'); END;"
960+
"CREATE INDEX IF NOT EXISTS delivery_request_results ON delivery_log(request_id,event_id,phase);";
961+
const char* const integrity =
962+
"CREATE TABLE IF NOT EXISTS report_integrity (input_cursor INTEGER PRIMARY KEY REFERENCES report_snapshots(input_cursor),digest TEXT NOT NULL);"
963+
"CREATE TRIGGER IF NOT EXISTS report_integrity_no_update BEFORE UPDATE ON report_integrity BEGIN SELECT RAISE(ABORT,'report integrity is immutable'); END;"
964+
"CREATE TRIGGER IF NOT EXISTS report_integrity_no_delete BEFORE DELETE ON report_integrity BEGIN SELECT RAISE(ABORT,'report integrity is immutable'); END;";
965+
// The first runner's tables, then webhook routing, cumulative reports,
966+
// redelivery requests and integrity digests; the current runner adds the
967+
// delivery summaries.
968+
const std::vector<std::pair<const char*, const char*>> stages{
969+
{"3", first}, {"6", routing}, {"7", reports}, {"8", redelivery}, {"9", integrity}};
970+
for (std::size_t written = 1; written <= stages.size(); ++written) {
971+
const auto* tables = stages[written - 1].first;
972+
const auto path = temp.file(std::string(tables) + "-tables.sqlite");
973+
for (std::size_t stage = 0; stage < written; ++stage) execute_sql(path, stages[stage].second);
974+
CHECK(table_count(path) == tables);
975+
const auto refusal = open_refusal(path, "layouts");
976+
if (!refusal.empty()) std::fprintf(stderr, "%s-table layout refused: %s\n", tables, refusal.c_str());
977+
CHECK(refusal.empty());
978+
CHECK(table_count(path) == "12");
979+
CHECK(open_refusal(path, "layouts").empty());
980+
}
981+
const auto current = temp.file("12-tables.sqlite");
982+
{ Ledger ledger(current, "layouts"); }
983+
CHECK(table_count(current) == "12");
984+
CHECK(open_refusal(current, "layouts").empty());
985+
986+
const std::string unsupported = "native ledger is not an empty or supported ledger database";
987+
const char* const unrelated = "CREATE TABLE unrelated (value TEXT);";
988+
const auto refused = [&](const char* tables, const std::vector<const char*>& statements) {
989+
const auto path = temp.file(std::string("refused-") + tables + ".sqlite");
990+
for (const auto* statement : statements) execute_sql(path, statement);
991+
CHECK(table_count(path) == tables);
992+
CHECK(open_refusal(path, "layouts") == unsupported);
993+
CHECK(table_count(path) == tables);
994+
};
995+
refused("4", {first, unrelated});
996+
refused("5", {first, unrelated, "CREATE TABLE other (value TEXT);"});
997+
refused("10", {first, routing, reports, redelivery, integrity, unrelated});
998+
execute_sql(current, unrelated);
999+
CHECK(table_count(current) == "13");
1000+
CHECK(open_refusal(current, "layouts") == unsupported);
1001+
CHECK(table_count(current) == "13");
1002+
}
1003+
8661004
void ledger_transactions() {
8671005
TempDir temp;
8681006
const auto path = temp.file("orders.sqlite");
@@ -1156,6 +1294,8 @@ int main(int argc, char** argv) {
11561294
delivery_summary_migration();
11571295
ledger_transactions();
11581296
cumulative_reports();
1297+
pre_integrity_ledger();
1298+
ledger_layouts();
11591299
incremental_reports();
11601300
crash_recovery(argv[0]);
11611301
native_http();

0 commit comments

Comments
 (0)