Skip to content

Commit 461fe0e

Browse files
lun3xritchie46
authored andcommitted
feat(python): Add a workspace argument to Config.enable_monitoring (#29196)
(cherry picked from commit fc2c4e1)
1 parent 0ced41b commit 461fe0e

6 files changed

Lines changed: 103 additions & 15 deletions

File tree

‎crates/polars-python/src/polars_cloud_observer.rs‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,11 @@ impl QueryObserver for PolarsCloudObserver {
133133
}
134134
}
135135

136+
/// Register the `polars_cloud` query observer. `workspace` selects the workspace query
137+
/// metrics are sent to, given as a name or an id; `None` uses the default workspace.
136138
#[pyfunction]
137-
pub fn set_query_monitoring(py: Python<'_>, enable: bool) -> PyResult<()> {
139+
#[pyo3(signature = (enable, workspace=None))]
140+
pub fn set_query_monitoring(py: Python<'_>, enable: bool, workspace: Option<&str>) -> PyResult<()> {
138141
if !enable {
139142
register_query_observer_factory(None);
140143
return Ok(());
@@ -154,14 +157,18 @@ pub fn set_query_monitoring(py: Python<'_>, enable: bool) -> PyResult<()> {
154157
Ensure the polars_cloud and polars versions match.",
155158
)
156159
})?;
157-
let observer = cls
158-
.call0()
159-
.map_err(|e| {
160-
PyRuntimeError::new_err(format!(
161-
"failed to construct the Polars Cloud observer: {e}"
162-
))
163-
})?
164-
.unbind();
160+
let observer = match workspace {
161+
// Passing no argument keeps working with `polars_cloud` versions whose observer
162+
// does not take a workspace.
163+
None => cls.call0(),
164+
Some(workspace) => cls.call1((workspace,)),
165+
}
166+
.map_err(|e| {
167+
PyRuntimeError::new_err(format!(
168+
"failed to construct the Polars Cloud observer: {e}"
169+
))
170+
})?
171+
.unbind();
165172

166173
register_query_observer_factory(Some(Arc::new(CloudObserverFactory {
167174
observer: Arc::new(observer),

‎py-polars/src/polars/_plr.pyi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2399,7 +2399,7 @@ def check_length(check: bool) -> None: ...
23992399
def get_engine_affinity() -> EngineType: ...
24002400
def config_reload_env_vars() -> None: ...
24012401
def config_reload_env_var(var: str) -> None: ...
2402-
def set_query_monitoring(enable: bool) -> None: ...
2402+
def set_query_monitoring(enable: bool, workspace: str | None = None) -> None: ...
24032403

24042404
# functions.when
24052405
class PyWhen:

‎py-polars/src/polars/_utils/monitoring.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,18 @@
66
from typing import Final
77

88
MONITORING_ENV_VAR: Final[str] = "POLARS_QUERY_MONITORING"
9+
MONITORING_WORKSPACE_ENV_VAR: Final[str] = "POLARS_QUERY_MONITORING_WORKSPACE"
910

1011

1112
def monitoring_enabled_globally() -> bool:
1213
return os.environ.get(MONITORING_ENV_VAR) == "1"
1314

1415

16+
def monitoring_workspace() -> str | None:
17+
"""The Polars Cloud workspace metrics are sent to, `None` for the default one."""
18+
return os.environ.get(MONITORING_WORKSPACE_ENV_VAR) or None
19+
20+
1521
def activate_monitoring() -> None:
1622
"""Ensure `polars_cloud` is installed and this session is authenticated."""
1723
from polars._dependencies import import_optional

‎py-polars/src/polars/config.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77

88
from polars._dependencies import json
99
from polars._utils.deprecation import deprecated
10-
from polars._utils.monitoring import MONITORING_ENV_VAR, activate_monitoring
10+
from polars._utils.monitoring import (
11+
MONITORING_ENV_VAR,
12+
MONITORING_WORKSPACE_ENV_VAR,
13+
activate_monitoring,
14+
)
1115
from polars._utils.unstable import unstable
1216
from polars._utils.various import normalize_filepath
1317
from polars.lazyframe.engine import Engine
@@ -85,6 +89,7 @@
8589
"POLARS_MAX_EXPR_DEPTH",
8690
"POLARS_ENGINE_AFFINITY",
8791
"POLARS_QUERY_MONITORING",
92+
"POLARS_QUERY_MONITORING_WORKSPACE",
8893
}
8994

9095
# vars that set the rust env directly should declare themselves here as the Config
@@ -1632,7 +1637,9 @@ def set_engine_affinity(cls, engine: EngineType | None = None) -> type[Config]:
16321637
return cls
16331638

16341639
@classmethod
1635-
def enable_monitoring(cls, active: bool | None = True) -> type[Config]:
1640+
def enable_monitoring(
1641+
cls, active: bool | None = True, *, workspace: str | None = None
1642+
) -> type[Config]:
16361643
"""
16371644
Enable runtime monitoring of query execution.
16381645
@@ -1656,11 +1663,19 @@ def enable_monitoring(cls, active: bool | None = True) -> type[Config]:
16561663
----------
16571664
active
16581665
Enable monitoring when True (the default), disable it when False.
1666+
workspace
1667+
Name or id of the Polars Cloud workspace the metrics are sent to; defaults
1668+
to the default workspace of your account. Ignored when disabling
1669+
monitoring.
16591670
16601671
Examples
16611672
--------
16621673
>>> pl.Config.enable_monitoring() # doctest: +SKIP
16631674
1675+
Send the metrics to a specific workspace instead of the default one:
1676+
1677+
>>> pl.Config.enable_monitoring(workspace="my-workspace") # doctest: +SKIP
1678+
16641679
Enable monitoring temporarily with ``Config``; the previous monitoring state
16651680
and engine affinity are restored on exit:
16661681
@@ -1671,9 +1686,14 @@ def enable_monitoring(cls, active: bool | None = True) -> type[Config]:
16711686
activate_monitoring()
16721687

16731688
os.environ[MONITORING_ENV_VAR] = "1"
1689+
if workspace is None:
1690+
os.environ.pop(MONITORING_WORKSPACE_ENV_VAR, None)
1691+
else:
1692+
os.environ[MONITORING_WORKSPACE_ENV_VAR] = workspace
16741693
cls.set_engine_affinity("streaming")
16751694
else:
16761695
os.environ.pop(MONITORING_ENV_VAR, None)
1696+
os.environ.pop(MONITORING_WORKSPACE_ENV_VAR, None)
16771697

16781698
return cls
16791699

‎py-polars/src/polars/lazyframe/engine.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,10 @@ def name(self) -> str:
354354

355355
def _with_monitoring(self, optimizations: QueryOptFlags) -> QueryOptFlags:
356356
"""Register the query observer when enabled and update `optimizations`."""
357-
from polars._utils.monitoring import monitoring_enabled_globally
357+
from polars._utils.monitoring import (
358+
monitoring_enabled_globally,
359+
monitoring_workspace,
360+
)
358361

359362
monitor = (
360363
monitoring_enabled_globally()
@@ -364,7 +367,7 @@ def _with_monitoring(self, optimizations: QueryOptFlags) -> QueryOptFlags:
364367
if monitor:
365368
import polars._plr as plr
366369

367-
plr.set_query_monitoring(True)
370+
plr.set_query_monitoring(True, monitoring_workspace())
368371

369372
optimizations = optimizations.__copy__()
370373
optimizations._pyoptflags.query_monitoring = monitor

‎py-polars/tests/unit/lazyframe/test_query_monitoring.py‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414

1515
import polars as pl
1616
import polars._plr as plr
17-
from polars._utils.monitoring import MONITORING_ENV_VAR
17+
from polars._utils.monitoring import (
18+
MONITORING_ENV_VAR,
19+
MONITORING_WORKSPACE_ENV_VAR,
20+
)
1821
from polars.lazyframe.engine import StreamingEngine
1922
from tests.unit.conftest import mock_module_import
2023

@@ -70,6 +73,55 @@ def test_config_enable_monitoring() -> None:
7073
assert observer.on_query_started.call_count == 2
7174

7275

76+
def test_config_enable_monitoring_workspace() -> None:
77+
"""The configured workspace is handed to the observer on construction."""
78+
module, _ = fake_cloud_observer()
79+
with mock_module_import("polars_cloud", module, replace_if_exists=True):
80+
pl.Config.enable_monitoring(workspace="my-workspace")
81+
assert os.environ[MONITORING_WORKSPACE_ENV_VAR] == "my-workspace"
82+
83+
_sample_lf().collect()
84+
module.QueryCloudObserver.assert_called_with("my-workspace")
85+
86+
# re-enabling without a workspace goes back to the default workspace
87+
pl.Config.enable_monitoring()
88+
assert MONITORING_WORKSPACE_ENV_VAR not in os.environ
89+
90+
_sample_lf().collect()
91+
module.QueryCloudObserver.assert_called_with()
92+
93+
# disabling monitoring clears the workspace as well
94+
pl.Config.enable_monitoring(workspace="my-workspace")
95+
pl.Config.enable_monitoring(False)
96+
assert MONITORING_WORKSPACE_ENV_VAR not in os.environ
97+
98+
99+
def test_config_scope_monitoring_workspace() -> None:
100+
"""The workspace is restored together with the other Config options."""
101+
module, _ = fake_cloud_observer()
102+
with mock_module_import("polars_cloud", module, replace_if_exists=True):
103+
pl.Config.enable_monitoring(workspace="outer-workspace")
104+
105+
with pl.Config():
106+
pl.Config.enable_monitoring(workspace="inner-workspace")
107+
_sample_lf().collect()
108+
module.QueryCloudObserver.assert_called_with("inner-workspace")
109+
110+
assert os.environ[MONITORING_WORKSPACE_ENV_VAR] == "outer-workspace"
111+
_sample_lf().collect()
112+
module.QueryCloudObserver.assert_called_with("outer-workspace")
113+
114+
115+
def test_engine_monitoring_uses_configured_workspace() -> None:
116+
"""An engine-level override still reports to the configured workspace."""
117+
module, _ = fake_cloud_observer()
118+
with mock_module_import("polars_cloud", module, replace_if_exists=True):
119+
pl.Config.enable_monitoring(workspace="my-workspace")
120+
_sample_lf().collect(engine=StreamingEngine(monitoring=True))
121+
122+
module.QueryCloudObserver.assert_called_with("my-workspace")
123+
124+
73125
def test_collect_calls_observer() -> None:
74126
"""Running a query with monitoring on invokes the observer callbacks."""
75127
module, observer = fake_cloud_observer()

0 commit comments

Comments
 (0)