Fix race condition and silent failure in bounded prometheus series tracker

cursoragent · cursoragent · commit bede81b2b937 · 2026-05-06T03:41:58.000Z
- Hold tracker lock around metric.labels() and track_series() so a
  concurrent eviction cannot remove the just-created Prometheus child
  before the caller observes/increments it.
- Only commit tracker state changes (popitem / del) when the underlying
  metric.remove() call actually succeeds (or the child was already
  gone), so silent removal failures cannot let cardinality drift above
  max_series.
diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py
@@ -994,9 +994,10 @@ def _get_labeled_metric(
         metric_name: DEFINED_PROMETHEUS_METRICS,
         labels: Dict[str, Optional[str]],
     ) -> Any:
-        labeled_metric = metric.labels(**labels)
-        self._track_bounded_prometheus_metric_series(metric, metric_name, labels)
-        return labeled_metric
+        with self._bounded_prometheus_series_tracker.lock:
+            labeled_metric = metric.labels(**labels)
+            self._track_bounded_prometheus_metric_series(metric, metric_name, labels)
+            return labeled_metric
 
     def _track_bounded_prometheus_metric_series(
         self,
diff --git a/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py b/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py
@@ -17,7 +17,7 @@ class BoundedPrometheusSeriesTracker:
     def __init__(self) -> None:
         self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
         self._last_ttl_cleanup: Dict[str, float] = {}
-        self._lock = RLock()
+        self.lock = RLock()
 
     def track_series(
         self,
@@ -33,7 +33,7 @@ def track_series(
 
         now = time.monotonic()
 
-        with self._lock:
+        with self.lock:
             series = self._series.setdefault(metric_name, OrderedDict())
             series[label_values] = now
             series.move_to_end(label_values)
@@ -53,12 +53,16 @@ def track_series(
 
             if max_series is not None and max_series > 0:
                 while len(series) > max_series:
-                    tracked_label_values, _ = series.popitem(last=False)
-                    self._remove_metric_child(metric, tracked_label_values)
+                    tracked_label_values = next(iter(series))
+                    if not self._remove_metric_child(metric, tracked_label_values):
+                        break
+                    del series[tracked_label_values]
             elif max_series is not None:
                 while series:
-                    tracked_label_values, _ = series.popitem(last=False)
-                    self._remove_metric_child(metric, tracked_label_values)
+                    tracked_label_values = next(iter(series))
+                    if not self._remove_metric_child(metric, tracked_label_values):
+                        break
+                    del series[tracked_label_values]
 
     def _should_run_ttl_cleanup(
         self,
@@ -82,15 +86,26 @@ def _remove_metric_series(
         series: OrderedDict[tuple[Optional[str], ...], float],
         label_values: tuple[Optional[str], ...],
     ) -> None:
-        if label_values in series:
-            del series[label_values]
-        self._remove_metric_child(metric, label_values)
+        if self._remove_metric_child(metric, label_values):
+            series.pop(label_values, None)
 
     @staticmethod
     def _remove_metric_child(
         metric: Any, label_values: tuple[Optional[str], ...]
-    ) -> None:
+    ) -> bool:
+        """
+        Remove the Prometheus child for ``label_values`` and report whether the
+        tracker should commit the matching state change.
+
+        Returns ``True`` when the child is no longer present in Prometheus
+        (either it was just removed or it was already gone), and ``False`` when
+        ``metric.remove()`` raised an unexpected error and the child likely
+        still exists.
+        """
         try:
             metric.remove(*label_values)
-        except (AttributeError, KeyError, ValueError):
-            pass
+            return True
+        except KeyError:
+            return True
+        except (AttributeError, ValueError):
+            return False
