121121MODEL_TIER = "tiny"
122122
123123
124- def _resolve_weights () -> str | None :
125- """Return the checkpoint path when weights are on disk, else None ."""
124+ def _weights_on_disk () -> bool :
125+ """Whether a checkpoint with weights exists anywhere the resolver looks ."""
126126 try :
127127 from unplug .ml .validation import resolve_validation_checkpoint
128128 except ImportError :
129- return None
129+ return False
130130 try :
131- path = resolve_validation_checkpoint (require_weights = True )
131+ return resolve_validation_checkpoint (require_weights = True ) is not None
132132 except Exception :
133- return None
134- return str ( path ) if path else None
133+ return False
134+
135135
136+ def _release_guard () -> Guard | None :
137+ """A guard configured the way a release runs, or None if it cannot load.
136138
137- def _guard_factory (weights : str | None ) -> Callable [[], Guard ] | None :
138- """Build guards the way a release runs, or None to take evaluate's default."""
139- if weights is None :
139+ Returned rather than described, because the checkpoint the report names has
140+ to be the checkpoint that scores. The resolver reads UNPLUG_TEST_CHECKPOINT
141+ before UNPLUG_MODEL_PATH before the cache, and Guard resolves separately, so
142+ two lookups can disagree and the run would name one checkpoint while
143+ measuring another. Reading it back off the loaded guard removes the second
144+ lookup.
145+ """
146+ if not _weights_on_disk ():
147+ return None
148+ try :
149+ guard = Guard (model = MODEL_TIER , require_ml = True )
150+ except Exception :
140151 return None
141- return lambda : Guard ( model = MODEL_TIER , require_ml = True )
152+ return guard if guard . ml_model_loaded else None
142153
143154
144155def run_gate (threshold : float = 0.5 , * , require_weights : bool = False ) -> tuple [bool , dict ]:
@@ -147,33 +158,39 @@ def run_gate(threshold: float = 0.5, *, require_weights: bool = False) -> tuple[
147158
148159 # Resolved once: every benign slice below must be measured under the same
149160 # configuration, or the numbers in one run are not comparable to each other.
150- weights = _resolve_weights ()
151- factory = _guard_factory (weights )
161+ # One configuration for the whole run, so the report names the checkpoint
162+ # that actually scored and every check below is measured the same way.
163+ shared = _release_guard ()
164+ factory : Callable [[], Guard ] | None = (
165+ None if shared is None else (lambda : Guard (model = MODEL_TIER , require_ml = True ))
166+ )
152167 # Every slice is scored with isolated_requests: one guard per slice, a fresh
153168 # ExecutionContext per sample. The default rebuilds the guard per sample,
154169 # which reloads the checkpoint 95 times and turns a 6 second slice into 27.
155170 # Verified to give identical counts on all four slices.
156171 report ["model" ] = {
157172 "tier" : MODEL_TIER ,
158- "weights_present" : weights is not None ,
159- "checkpoint" : weights ,
160- "measured_with" : "ml" if weights else "regex_only" ,
173+ "weights_present" : shared is not None ,
174+ "checkpoint" : None if shared is None else shared . active_checkpoint ,
175+ "measured_with" : "ml" if shared is not None else "regex_only" ,
161176 "required" : require_weights ,
162177 }
163- if require_weights and weights is None :
178+ if require_weights and shared is None :
164179 passed = False
165180
166- matrix = run_matrix (threshold = threshold )
181+ # The matrix builds a fresh guard per scan by design, so it takes the factory
182+ # rather than the shared instance. Without it the matrix ran on a default
183+ # Guard() while the report above said the run was measured with the model,
184+ # which was true of the slices and not of the matrix.
185+ matrix = run_matrix (threshold = threshold , guard_factory = factory )
167186 report ["converter_matrix" ] = matrix .to_dict ()
168187 if not matrix .passed :
169188 passed = False
170189
171190 corpus_report : dict = {"floors" : {}, "shortfalls" : []}
172191 if GARAK_CORPUS .exists ():
173192 samples = load_jsonl (GARAK_CORPUS )
174- result = evaluate (
175- samples , threshold = threshold , guard_factory = factory , isolated_requests = True
176- )
193+ result = evaluate (samples , threshold = threshold , guard = shared , isolated_requests = True )
177194 for category , floor in GARAK_RECALL_FLOORS .items ():
178195 metrics = result .by_category .get (category )
179196 recall = metrics .recall if metrics else 0.0
@@ -204,9 +221,7 @@ def run_gate(threshold: float = 0.5, *, require_weights: bool = False) -> tuple[
204221 # there: a corpus with only hard rows, and a labelling change that moves
205222 # the easy rows off label=0.
206223 easy_missing = not easy
207- easy_result = evaluate (
208- easy , threshold = threshold , guard_factory = factory , isolated_requests = True
209- )
224+ easy_result = evaluate (easy , threshold = threshold , guard = shared , isolated_requests = True )
210225 easy_fpr = easy_result .overall .false_positive_rate
211226 easy_ok = (easy_fpr <= EASY_FPR_CEILING ) if easy else False
212227
@@ -216,7 +231,7 @@ def run_gate(threshold: float = 0.5, *, require_weights: bool = False) -> tuple[
216231 # target was met while it was at it.
217232 hard_missing = not hard
218233 hard_result = (
219- evaluate (hard , threshold = threshold , guard_factory = factory , isolated_requests = True )
234+ evaluate (hard , threshold = threshold , guard = shared , isolated_requests = True )
220235 if hard
221236 else None
222237 )
@@ -263,14 +278,12 @@ def run_gate(threshold: float = 0.5, *, require_weights: bool = False) -> tuple[
263278 carrier_samples = [s for s in load_jsonl (CARRIER_CORPUS ) if s .label == 0 ]
264279 carrier_missing = not carrier_samples
265280 carrier_result = (
266- evaluate (
267- carrier_samples , threshold = threshold , guard_factory = factory , isolated_requests = True
268- )
281+ evaluate (carrier_samples , threshold = threshold , guard = shared , isolated_requests = True )
269282 if carrier_samples
270283 else None
271284 )
272285 carrier_fpr = carrier_result .overall .false_positive_rate if carrier_result else 0.0
273- measurable = carrier_result is not None and weights is not None
286+ measurable = carrier_result is not None and shared is not None
274287 carrier_ok = (carrier_fpr <= CARRIER_FPR_RATCHET ) if measurable else False
275288 carrier_report = {
276289 "samples" : len (carrier_samples ),
@@ -288,9 +301,13 @@ def run_gate(threshold: float = 0.5, *, require_weights: bool = False) -> tuple[
288301 "meets_target" : measurable and carrier_fpr <= CARRIER_FPR_TARGET ,
289302 "to_target" : round (max (0.0 , carrier_fpr - CARRIER_FPR_TARGET ), 4 ),
290303 }
291- # An unmeasurable slice does not fail the CI run, which has no weights by
292- # design. It fails --require-weights, which is where a release asks.
293- if measurable and not carrier_ok :
304+ # Three different states, and collapsing them is how a gate goes green
305+ # while measuring nothing. A corpus with no usable rows always fails:
306+ # someone emptied the file and no configuration excuses that. A slice
307+ # that could not be scored for want of weights does not fail the default
308+ # run, because CI has none by design, but it does fail
309+ # --require-weights. A scored slice fails on the ratchet.
310+ if carrier_missing or (measurable and not carrier_ok ):
294311 passed = False
295312 if require_weights and not measurable :
296313 passed = False
0 commit comments