Skip to content

Commit b069c83

Browse files
lun3xclaude
authored andcommitted
feat(python): Add organization to Config.enable_monitoring (#29221)
Co-authored-by: Claude Fable 5.1 <[email protected]> (cherry picked from commit bb38e0c)
1 parent d5a00ec commit b069c83

6 files changed

Lines changed: 117 additions & 23 deletions

File tree

‎crates/polars-python/src/polars_cloud_observer.rs‎

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use polars_observer::{
88
};
99
use pyo3::exceptions::PyRuntimeError;
1010
use pyo3::prelude::*;
11-
use pyo3::types::PyBytes;
11+
use pyo3::types::{PyBytes, PyDict};
1212
use uuid::Uuid;
1313

1414
const POLARS_CLOUD_PACKAGE_NAME: &str = "polars_cloud";
@@ -133,11 +133,17 @@ impl QueryObserver for PolarsCloudObserver {
133133
}
134134
}
135135

136-
/// Register the `polars_cloud` query observer. `workspace` selects the workspace query
137-
/// metrics are sent to, given as a name or an id; `None` uses the default workspace.
136+
/// Register the `polars_cloud` query observer. `workspace` and `organization` select
137+
/// where query metrics are sent to, each given as a name or an id; `None` uses the
138+
/// default workspace or organization of the account.
138139
#[pyfunction]
139-
#[pyo3(signature = (enable, workspace=None))]
140-
pub fn set_query_monitoring(py: Python<'_>, enable: bool, workspace: Option<&str>) -> PyResult<()> {
140+
#[pyo3(signature = (enable, workspace=None, organization=None))]
141+
pub fn set_query_monitoring(
142+
py: Python<'_>,
143+
enable: bool,
144+
workspace: Option<&str>,
145+
organization: Option<&str>,
146+
) -> PyResult<()> {
141147
if !enable {
142148
register_query_observer_factory(None);
143149
return Ok(());
@@ -157,18 +163,23 @@ pub fn set_query_monitoring(py: Python<'_>, enable: bool, workspace: Option<&str
157163
Ensure the polars_cloud and polars versions match.",
158164
)
159165
})?;
160-
let observer = match workspace {
161-
// Passing no argument keeps working with `polars_cloud` versions whose observer
162-
// does not take a workspace.
163-
None => cls.call0(),
164-
Some(workspace) => cls.call1((workspace,)),
166+
// Only the arguments that are set are passed, as keywords, so that older
167+
// `polars_cloud` versions whose observer does not take them keep working.
168+
let kwargs = PyDict::new(py);
169+
if let Some(workspace) = workspace {
170+
kwargs.set_item("workspace", workspace)?;
165171
}
166-
.map_err(|e| {
167-
PyRuntimeError::new_err(format!(
168-
"failed to construct the Polars Cloud observer: {e}"
169-
))
170-
})?
171-
.unbind();
172+
if let Some(organization) = organization {
173+
kwargs.set_item("organization", organization)?;
174+
}
175+
let observer = cls
176+
.call((), Some(&kwargs))
177+
.map_err(|e| {
178+
PyRuntimeError::new_err(format!(
179+
"failed to construct the Polars Cloud observer: {e}"
180+
))
181+
})?
182+
.unbind();
172183

173184
register_query_observer_factory(Some(Arc::new(CloudObserverFactory {
174185
observer: Arc::new(observer),

‎py-polars/src/polars/_plr.pyi‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2399,7 +2399,9 @@ def check_length(check: bool) -> None: ...
23992399
def get_engine_affinity() -> EngineType: ...
24002400
def config_reload_env_vars() -> None: ...
24012401
def config_reload_env_var(var: str) -> None: ...
2402-
def set_query_monitoring(enable: bool, workspace: str | None = None) -> None: ...
2402+
def set_query_monitoring(
2403+
enable: bool, workspace: str | None = None, organization: str | None = None
2404+
) -> None: ...
24032405

24042406
# functions.when
24052407
class PyWhen:

‎py-polars/src/polars/_utils/monitoring.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
MONITORING_ENV_VAR: Final[str] = "POLARS_QUERY_MONITORING"
99
MONITORING_WORKSPACE_ENV_VAR: Final[str] = "POLARS_QUERY_MONITORING_WORKSPACE"
10+
MONITORING_ORGANIZATION_ENV_VAR: Final[str] = "POLARS_QUERY_MONITORING_ORGANIZATION"
1011

1112

1213
def monitoring_enabled_globally() -> bool:
@@ -18,6 +19,11 @@ def monitoring_workspace() -> str | None:
1819
return os.environ.get(MONITORING_WORKSPACE_ENV_VAR) or None
1920

2021

22+
def monitoring_organization() -> str | None:
23+
"""The Polars Cloud organization metrics are sent to, `None` for the default one."""
24+
return os.environ.get(MONITORING_ORGANIZATION_ENV_VAR) or None
25+
26+
2127
def activate_monitoring() -> None:
2228
"""Ensure `polars_cloud` is installed and this session is authenticated."""
2329
from polars._dependencies import import_optional

‎py-polars/src/polars/config.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from polars._utils.deprecation import deprecated
1010
from polars._utils.monitoring import (
1111
MONITORING_ENV_VAR,
12+
MONITORING_ORGANIZATION_ENV_VAR,
1213
MONITORING_WORKSPACE_ENV_VAR,
1314
activate_monitoring,
1415
)
@@ -90,6 +91,7 @@
9091
"POLARS_ENGINE_AFFINITY",
9192
"POLARS_QUERY_MONITORING",
9293
"POLARS_QUERY_MONITORING_WORKSPACE",
94+
"POLARS_QUERY_MONITORING_ORGANIZATION",
9395
}
9496

9597
# vars that set the rust env directly should declare themselves here as the Config
@@ -1638,7 +1640,11 @@ def set_engine_affinity(cls, engine: EngineType | None = None) -> type[Config]:
16381640

16391641
@classmethod
16401642
def enable_monitoring(
1641-
cls, active: bool | None = True, *, workspace: str | None = None
1643+
cls,
1644+
active: bool | None = True,
1645+
*,
1646+
workspace: str | None = None,
1647+
organization: str | None = None,
16421648
) -> type[Config]:
16431649
"""
16441650
Enable runtime monitoring of query execution.
@@ -1667,6 +1673,11 @@ def enable_monitoring(
16671673
Name or id of the Polars Cloud workspace the metrics are sent to; defaults
16681674
to the default workspace of your account. Ignored when disabling
16691675
monitoring.
1676+
organization
1677+
Name or id of the Polars Cloud organization the workspace belongs to;
1678+
defaults to the default organization of your account. Use it to
1679+
disambiguate a workspace name that exists in several organizations.
1680+
Ignored when disabling monitoring.
16701681
16711682
Examples
16721683
--------
@@ -1676,6 +1687,12 @@ def enable_monitoring(
16761687
16771688
>>> pl.Config.enable_monitoring(workspace="my-workspace") # doctest: +SKIP
16781689
1690+
Select the organization as well when the workspace name is not unique:
1691+
1692+
>>> pl.Config.enable_monitoring(
1693+
... workspace="my-workspace", organization="my-org"
1694+
... ) # doctest: +SKIP
1695+
16791696
Enable monitoring temporarily with ``Config``; the previous monitoring state
16801697
and engine affinity are restored on exit:
16811698
@@ -1690,10 +1707,15 @@ def enable_monitoring(
16901707
os.environ.pop(MONITORING_WORKSPACE_ENV_VAR, None)
16911708
else:
16921709
os.environ[MONITORING_WORKSPACE_ENV_VAR] = workspace
1710+
if organization is None:
1711+
os.environ.pop(MONITORING_ORGANIZATION_ENV_VAR, None)
1712+
else:
1713+
os.environ[MONITORING_ORGANIZATION_ENV_VAR] = organization
16931714
cls.set_engine_affinity("streaming")
16941715
else:
16951716
os.environ.pop(MONITORING_ENV_VAR, None)
16961717
os.environ.pop(MONITORING_WORKSPACE_ENV_VAR, None)
1718+
os.environ.pop(MONITORING_ORGANIZATION_ENV_VAR, None)
16971719

16981720
return cls
16991721

‎py-polars/src/polars/lazyframe/engine.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,7 @@ def _with_monitoring(self, optimizations: QueryOptFlags) -> QueryOptFlags:
356356
"""Register the query observer when enabled and update `optimizations`."""
357357
from polars._utils.monitoring import (
358358
monitoring_enabled_globally,
359+
monitoring_organization,
359360
monitoring_workspace,
360361
)
361362

@@ -367,7 +368,11 @@ def _with_monitoring(self, optimizations: QueryOptFlags) -> QueryOptFlags:
367368
if monitor:
368369
import polars._plr as plr
369370

370-
plr.set_query_monitoring(True, monitoring_workspace())
371+
plr.set_query_monitoring(
372+
True,
373+
workspace=monitoring_workspace(),
374+
organization=monitoring_organization(),
375+
)
371376

372377
optimizations = optimizations.__copy__()
373378
optimizations._pyoptflags.query_monitoring = monitor

‎py-polars/tests/unit/lazyframe/test_query_monitoring.py‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import polars._plr as plr
1717
from polars._utils.monitoring import (
1818
MONITORING_ENV_VAR,
19+
MONITORING_ORGANIZATION_ENV_VAR,
1920
MONITORING_WORKSPACE_ENV_VAR,
2021
)
2122
from polars.lazyframe.engine import StreamingEngine
@@ -81,7 +82,7 @@ def test_config_enable_monitoring_workspace() -> None:
8182
assert os.environ[MONITORING_WORKSPACE_ENV_VAR] == "my-workspace"
8283

8384
_sample_lf().collect()
84-
module.QueryCloudObserver.assert_called_with("my-workspace")
85+
module.QueryCloudObserver.assert_called_with(workspace="my-workspace")
8586

8687
# re-enabling without a workspace goes back to the default workspace
8788
pl.Config.enable_monitoring()
@@ -96,6 +97,53 @@ def test_config_enable_monitoring_workspace() -> None:
9697
assert MONITORING_WORKSPACE_ENV_VAR not in os.environ
9798

9899

100+
def test_config_enable_monitoring_organization() -> None:
101+
"""The configured organization is handed to the observer on construction."""
102+
module, _ = fake_cloud_observer()
103+
with mock_module_import("polars_cloud", module, replace_if_exists=True):
104+
pl.Config.enable_monitoring(organization="my-org")
105+
assert os.environ[MONITORING_ORGANIZATION_ENV_VAR] == "my-org"
106+
assert MONITORING_WORKSPACE_ENV_VAR not in os.environ
107+
108+
_sample_lf().collect()
109+
module.QueryCloudObserver.assert_called_with(organization="my-org")
110+
111+
# workspace and organization are passed together
112+
pl.Config.enable_monitoring(workspace="my-workspace", organization="my-org")
113+
_sample_lf().collect()
114+
module.QueryCloudObserver.assert_called_with(
115+
workspace="my-workspace", organization="my-org"
116+
)
117+
118+
# re-enabling without an organization goes back to the default organization
119+
pl.Config.enable_monitoring()
120+
assert MONITORING_ORGANIZATION_ENV_VAR not in os.environ
121+
122+
_sample_lf().collect()
123+
module.QueryCloudObserver.assert_called_with()
124+
125+
# disabling monitoring clears the organization as well
126+
pl.Config.enable_monitoring(organization="my-org")
127+
pl.Config.enable_monitoring(False)
128+
assert MONITORING_ORGANIZATION_ENV_VAR not in os.environ
129+
130+
131+
def test_config_scope_monitoring_organization() -> None:
132+
"""The organization is restored together with the other Config options."""
133+
module, _ = fake_cloud_observer()
134+
with mock_module_import("polars_cloud", module, replace_if_exists=True):
135+
pl.Config.enable_monitoring(organization="outer-org")
136+
137+
with pl.Config():
138+
pl.Config.enable_monitoring(organization="inner-org")
139+
_sample_lf().collect()
140+
module.QueryCloudObserver.assert_called_with(organization="inner-org")
141+
142+
assert os.environ[MONITORING_ORGANIZATION_ENV_VAR] == "outer-org"
143+
_sample_lf().collect()
144+
module.QueryCloudObserver.assert_called_with(organization="outer-org")
145+
146+
99147
def test_config_scope_monitoring_workspace() -> None:
100148
"""The workspace is restored together with the other Config options."""
101149
module, _ = fake_cloud_observer()
@@ -105,11 +153,11 @@ def test_config_scope_monitoring_workspace() -> None:
105153
with pl.Config():
106154
pl.Config.enable_monitoring(workspace="inner-workspace")
107155
_sample_lf().collect()
108-
module.QueryCloudObserver.assert_called_with("inner-workspace")
156+
module.QueryCloudObserver.assert_called_with(workspace="inner-workspace")
109157

110158
assert os.environ[MONITORING_WORKSPACE_ENV_VAR] == "outer-workspace"
111159
_sample_lf().collect()
112-
module.QueryCloudObserver.assert_called_with("outer-workspace")
160+
module.QueryCloudObserver.assert_called_with(workspace="outer-workspace")
113161

114162

115163
def test_engine_monitoring_uses_configured_workspace() -> None:
@@ -119,7 +167,7 @@ def test_engine_monitoring_uses_configured_workspace() -> None:
119167
pl.Config.enable_monitoring(workspace="my-workspace")
120168
_sample_lf().collect(engine=StreamingEngine(monitoring=True))
121169

122-
module.QueryCloudObserver.assert_called_with("my-workspace")
170+
module.QueryCloudObserver.assert_called_with(workspace="my-workspace")
123171

124172

125173
def test_collect_calls_observer() -> None:

0 commit comments

Comments
 (0)