Skip to content

Commit b180338

Browse files
committed
docs(security): re-sync published pipeline copies to the hardened revision
Addresses @IvanHunters round-2 review of the published copies: - re-sync report.py, monthly.py, backfill_issues.py, sync_issues.py and scan.sh to security-scanner#1004, and add the new statelib.py, so the HIGH-path guard, the shared crash-safe state I/O across all four readers, review_after enforcement, the persisted age-based auto-dismissal and the scan.sh env-export fix are all present in the copies an auditor reads — not just in one call site. - README now names the exact source revision the copies correspond to (security-scanner#1004, commit b40462c1) and lists statelib.py. Once #1004 merges these match security-scanner main, the revision that runs on schedule. Signed-off-by: Timur Tukaev <[email protected]>
1 parent 7465482 commit b180338

7 files changed

Lines changed: 136 additions & 70 deletions

File tree

‎docs/security/pipeline/README.md‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ scripts, so that its method is auditable independently of the finding data it pr
55
The **source of truth is the private `cozystack/security-scanner` repository**, which also
66
holds the per-finding reports and triage state that are **not** published here.
77

8+
These copies correspond to `cozystack/security-scanner` PR #1004 (commit `b40462c1`), the
9+
revision that carries the state-integrity, HIGH-path and scan-failure hardening described
10+
below. They are kept byte-for-byte in step with it; once #1004 merges they match
11+
`security-scanner` `main`, the revision that runs on schedule.
12+
813
## Contents
914

1015
- `SECURITY_PIPELINE_POLICY.md` — what the pipeline scans, on what schedule, and what is
@@ -14,9 +19,10 @@ holds the per-finding reports and triage state that are **not** published here.
1419
identifiers in it are illustrative placeholders; real triage decisions live in the
1520
private state.
1621
- `scripts/` — the discovery, scanning and reporting scripts (`discover.sh`, `scan.sh`,
17-
`report.py`, `monthly.py`, `sync_issues.py`, `backfill_issues.py`), published so the
18-
method can be audited. These are reference copies; the runnable versions live in the
19-
private repository against organization-scoped credentials.
22+
`report.py`, `monthly.py`, `sync_issues.py`, `backfill_issues.py`, and `statelib.py`, the
23+
shared crash-safe state I/O the readers import), published so the method can be audited.
24+
These are reference copies; the runnable versions live in the private repository against
25+
organization-scoped credentials.
2026

2127
## Disclosure rule
2228

‎docs/security/pipeline/scripts/backfill_issues.py‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import time
2121
from datetime import datetime, timezone
2222

23+
from statelib import load_json, save_json
24+
2325
REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
2426
STATE_DIR = os.path.join(REPO_ROOT, "state")
2527
REPORTED_FILE = os.path.join(STATE_DIR, "reported-cves.json")
@@ -36,19 +38,6 @@
3638
UNFIXED_AGE_THRESHOLD_DAYS = 365
3739

3840

39-
def load_json(path, default):
40-
try:
41-
with open(path) as f:
42-
return json.load(f)
43-
except (FileNotFoundError, json.JSONDecodeError):
44-
return default
45-
46-
47-
def save_json(path, data):
48-
with open(path, "w") as f:
49-
json.dump(data, f, indent=2)
50-
51-
5241
def cve_age_days(cve_id):
5342
match = re.match(r"CVE-(\d{4})-", cve_id)
5443
if not match:

‎docs/security/pipeline/scripts/monthly.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,15 @@
1515
from collections import Counter
1616
from datetime import datetime, timezone, timedelta
1717

18+
from statelib import load_json
19+
1820
REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
1921
STATE_DIR = os.path.join(REPO_ROOT, "state")
2022
REPORTS_DIR = os.path.join(REPO_ROOT, "reports")
2123
REPORTED_FILE = os.path.join(STATE_DIR, "reported-cves.json")
2224
TRIAGE_FILE = os.path.join(STATE_DIR, "triage-overrides.json")
2325

2426

25-
def load_json(path, default):
26-
try:
27-
with open(path) as f:
28-
return json.load(f)
29-
except (FileNotFoundError, json.JSONDecodeError):
30-
return default
31-
32-
3327
def save_report(content, filename):
3428
dirpath = os.path.join(REPORTS_DIR, "monthly")
3529
os.makedirs(dirpath, exist_ok=True)

‎docs/security/pipeline/scripts/report.py‎

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
import os
1313
import re
1414
import subprocess
15-
from datetime import datetime, timezone
15+
from datetime import datetime, timedelta, timezone
16+
17+
from statelib import load_json, save_json
1618

1719

1820
REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
@@ -115,33 +117,6 @@ def notify_slack(vulns):
115117
print(f" WARN: Slack notification failed: {e}")
116118

117119

118-
def load_json(path, default):
119-
# A missing file is a legitimate first run -> default. A present-but-corrupt
120-
# file must NOT be silently treated as empty and then overwritten (that erases
121-
# the historic record — issue/advisory URLs, triage state). Fail loudly instead.
122-
if not os.path.exists(path):
123-
return default
124-
try:
125-
with open(path) as f:
126-
return json.load(f)
127-
except json.JSONDecodeError as e:
128-
raise SystemExit(
129-
f"FATAL: {path} exists but is not valid JSON ({e}). Refusing to "
130-
f"continue and overwrite it with an empty default — restore it from "
131-
f"git history (it may be a truncated write) and re-run."
132-
)
133-
134-
135-
def save_json(path, data):
136-
# Atomic write: a run killed mid-dump must not leave a truncated file that the
137-
# next run reads as empty. Write to a temp file, then rename over the target.
138-
os.makedirs(os.path.dirname(path), exist_ok=True)
139-
tmp = f"{path}.tmp"
140-
with open(tmp, "w") as f:
141-
json.dump(data, f, indent=2)
142-
os.replace(tmp, path)
143-
144-
145120
def cve_age_days(cve_id):
146121
"""Estimate CVE age from its ID (CVE-YYYY-NNNNN)."""
147122
match = re.match(r"CVE-(\d{4})-", cve_id)
@@ -178,6 +153,20 @@ def is_dev_only(vuln):
178153
return True
179154

180155

156+
def _review_after_passed(entry):
157+
"""True if the override carries a review_after date that is now due/past."""
158+
raw = entry.get("review_after")
159+
if not raw:
160+
return False
161+
try:
162+
due = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
163+
except ValueError:
164+
return False
165+
if due.tzinfo is None:
166+
due = due.replace(tzinfo=timezone.utc)
167+
return datetime.now(timezone.utc) >= due
168+
169+
181170
def classify_filter(vuln, triage_overrides):
182171
"""Classify a CVE for filtering. Returns (skip, reason) or (False, None).
183172
@@ -191,10 +180,14 @@ def classify_filter(vuln, triage_overrides):
191180
if severity == "UNKNOWN":
192181
return True, "unknown severity"
193182

194-
# Already triaged as resolved
183+
# Already triaged as resolved — UNLESS its review_after date has passed. A
184+
# suppression with an expired review date has done its job; keeping it would
185+
# dismiss the finding forever, which is exactly what review_after exists to
186+
# prevent. Let it surface again for re-review.
195187
if cve_id in triage_overrides:
196-
status = triage_overrides[cve_id].get("status", "")
197-
if status in RESOLVED_STATUSES:
188+
entry = triage_overrides[cve_id]
189+
status = entry.get("status", "")
190+
if status in RESOLVED_STATUSES and not _review_after_passed(entry):
198191
return True, f"triaged as {status}"
199192

200193
# Dev/build-only image or test component
@@ -207,6 +200,21 @@ def classify_filter(vuln, triage_overrides):
207200
if not vuln.get("fixed_version"):
208201
age = cve_age_days(cve_id)
209202
if age > UNFIXED_AGE_THRESHOLD_DAYS and severity not in ("CRITICAL", "HIGH"):
203+
# PERSIST the auto-dismissal as accepted-risk with a review_after date.
204+
# Returning skip alone records nothing, so the finding is re-filtered on
205+
# every run forever and never reaches the 90-day re-review the status
206+
# model requires. Writing it makes the drop auditable and time-boxed.
207+
if cve_id not in triage_overrides:
208+
now = datetime.now(timezone.utc)
209+
triage_overrides[cve_id] = {
210+
"status": "accepted-risk",
211+
"decided_by": "pipeline-auto",
212+
"decided_at": now.date().isoformat(),
213+
"review_after": (now + timedelta(days=90)).date().isoformat(),
214+
"reason": (f"[{vuln.get('package', '')}] Auto-demoted: {severity} finding with no "
215+
f"upstream fix, CVE approximately {age} days old (older than the "
216+
f"{UNFIXED_AGE_THRESHOLD_DAYS}-day threshold). Re-review on the date above."),
217+
}
210218
return True, f"unfixed upstream ({age} days old, {severity or 'no severity'})"
211219

212220
return False, None
@@ -587,6 +595,14 @@ def send_weekly_report(reported, pending):
587595
print(f"Creating issues for {len(high_vulns)} HIGH CVEs...")
588596
for vuln in high_vulns:
589597
issue_url = create_issue(vuln, report_path)
598+
# Same guard as the CRITICAL path: if the issue failed to create, do NOT
599+
# record the CVE as reported. Recording it with an empty issue_url makes the
600+
# skip check pass over it on every later run, so the 3-business-day triage
601+
# clock never starts. Leave it out so the next run retries it.
602+
if not issue_url:
603+
print(f" WARN: {vuln['cve_id']} — issue creation failed; not recording "
604+
f"as reported so the next run retries it")
605+
continue
590606
reported[vuln["cve_id"]] = {
591607
"severity": vuln["severity"],
592608
"reported_at": datetime.now(timezone.utc).isoformat(),
@@ -640,7 +656,11 @@ def main():
640656

641657
save_json(REPORTED_FILE, reported)
642658
save_json(PENDING_FILE, pending)
643-
print(f"\nState saved. Reported: {len(reported)}, Pending: {len(pending)}")
659+
# triage_overrides may have gained pipeline-auto accepted-risk entries from the
660+
# age filter; persist them so the dismissals are recorded and re-reviewed.
661+
save_json(TRIAGE_FILE, triage_overrides)
662+
print(f"\nState saved. Reported: {len(reported)}, Pending: {len(pending)}, "
663+
f"Triage overrides: {len(triage_overrides)}")
644664

645665

646666
if __name__ == "__main__":

‎docs/security/pipeline/scripts/scan.sh‎

100644100755
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ CLONE_DIR="$WORKSPACE/repos"
1010
RESULTS_DIR="$WORKSPACE/scan-results"
1111
OUTFILE="$WORKSPACE/scan-results.json"
1212

13+
# The Phase-3 merge runs as a heredoc Python subprocess and reads these paths from
14+
# the environment; a subprocess only inherits EXPORTED variables. Without this the
15+
# merge silently falls back to its hardcoded workspace/... defaults, so any run
16+
# invoked with a non-default WORKSPACE would merge the wrong directory.
17+
export WORKSPACE DISCOVERY IMAGES_FILE CLONE_DIR RESULTS_DIR OUTFILE
18+
1319
mkdir -p "$RESULTS_DIR"
1420

1521
if [ ! -f "$DISCOVERY" ]; then
@@ -152,6 +158,13 @@ for filename in sorted(os.listdir(results_dir)):
152158
except (json.JSONDecodeError, FileNotFoundError):
153159
continue
154160
161+
# A failed scan writes {"__scan_error__": true, "Results": []}. The run aborts
162+
# before this stage when a target fails, so reaching here with the marker means
163+
# a stale error file from an earlier run; do not merge it as a clean target.
164+
if data.get("__scan_error__"):
165+
print(f" WARN: skipping errored scan result {filename}", file=sys.stderr)
166+
continue
167+
155168
# Determine source (repo name or image name)
156169
source = filename.replace(".json", "")
157170
if source.startswith("repo-"):
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env python3
2+
"""Shared state I/O for the CVE-scanning pipeline.
3+
4+
The pipeline keeps its history in JSON files under state/ (reported-cves.json,
5+
pending-weekly.json, triage-overrides.json). Every script that touches them must
6+
read and write them the same crash-safe way, or one script's integrity guard is
7+
quietly undone by another's bare `except: return {}` and in-place write.
8+
9+
These two functions are the single implementation the readers import — report.py,
10+
monthly.py, backfill_issues.py and sync_issues.py — so there is no mirror to
11+
forget when the guard changes.
12+
"""
13+
import json
14+
import os
15+
import tempfile
16+
17+
18+
def load_json(path, default):
19+
"""Load JSON, or return default only for a genuinely missing file.
20+
21+
A missing file is a legitimate first run. A present-but-corrupt file must NOT
22+
be silently treated as empty and then overwritten: that erases the historic
23+
record (issue/advisory URLs, triage state), and a run killed mid-write leaves
24+
exactly such a truncated file. Fail loudly instead so it can be restored.
25+
"""
26+
if not os.path.exists(path):
27+
return default
28+
try:
29+
with open(path) as f:
30+
return json.load(f)
31+
except json.JSONDecodeError as e:
32+
raise SystemExit(
33+
f"FATAL: {path} exists but is not valid JSON ({e}). Refusing to "
34+
f"continue and overwrite it with an empty default — restore it from "
35+
f"git history (it may be a truncated write) and re-run."
36+
)
37+
38+
39+
def save_json(path, data):
40+
"""Write JSON atomically.
41+
42+
A run killed mid-dump must not leave a truncated file that the next run reads
43+
as empty. Write to a temp file in the same directory, then rename over the
44+
target (os.replace is atomic on POSIX).
45+
"""
46+
directory = os.path.dirname(path) or "."
47+
os.makedirs(directory, exist_ok=True)
48+
fd, tmp = tempfile.mkstemp(dir=directory, suffix=".tmp")
49+
try:
50+
with os.fdopen(fd, "w") as f:
51+
json.dump(data, f, indent=2)
52+
os.replace(tmp, path)
53+
except BaseException:
54+
if os.path.exists(tmp):
55+
os.unlink(tmp)
56+
raise

‎docs/security/pipeline/scripts/sync_issues.py‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import subprocess
1515
from datetime import datetime, timezone
1616

17+
from statelib import load_json, save_json
18+
1719
REPO_ROOT = os.path.join(os.path.dirname(__file__), "..")
1820
STATE_DIR = os.path.join(REPO_ROOT, "state")
1921
TRIAGE_FILE = os.path.join(STATE_DIR, "triage-overrides.json")
@@ -29,20 +31,6 @@
2931
}
3032

3133

32-
def load_json(path, default):
33-
try:
34-
with open(path) as f:
35-
return json.load(f)
36-
except (FileNotFoundError, json.JSONDecodeError):
37-
return default
38-
39-
40-
def save_json(path, data):
41-
os.makedirs(os.path.dirname(path), exist_ok=True)
42-
with open(path, "w") as f:
43-
json.dump(data, f, indent=2)
44-
45-
4634
def extract_cve_from_title(title):
4735
"""Extract CVE ID from issue title like 'security_critical: CVE-2025-15467 — ...'"""
4836
match = re.search(r"(CVE-\d{4}-\d+)", title)

0 commit comments

Comments
 (0)