Skip to content

Commit 5a2f2cd

Browse files
xrmxxrmx
authored
central config: hande disable_instrumentations (#510)
* Add handling for disable_instrumentations This implement disable_instrumentations using a rule based tracer configurator. * Fix typechecking * Fix and test default tracer_configurator * Address a macroscopeapp review * Add the tracers names for disable_instrumentations * Fix comparison of rules * Improve debug log * Guard against unexpected tracerprovider Spotted by macroscope
1 parent c3abe98 commit 5a2f2cd

8 files changed

Lines changed: 344 additions & 75 deletions

File tree

‎docs/reference/edot-python/configuration.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ You can modify the following settings for EDOT Python through APM Agent Central
100100
|---------------|----------------------------------------------|---------|---------|
101101
| Logging level | Configure EDOT Python agent logging level. | Dynamic | {applies_to}`stack: preview 9.1` <br> {applies_to}`edot_python: preview 1.4.0` |
102102
| Sampling rate | Configure EDOT Python tracing sampling rate. | Dynamic | {applies_to}`stack: preview 9.2` <br> {applies_to}`edot_python: preview 1.7.0` |
103+
| Deactivate instrumentations | Configure EDOT Python to deactivate tracers for specific instrumentations, supports globbing. Refer to [EDOT Python Supported technologies](/reference/edot-python/supported-technologies.md) for the names of the tracers. | Dynamic | {applies_to}`stack: preview 9.4` <br> {applies_to}`edot_python: preview 1.12.0` |
103104

104105
Dynamic settings can be changed without having to restart the application.
105106

‎docs/reference/edot-python/supported-technologies.md‎

Lines changed: 56 additions & 55 deletions
Large diffs are not rendered by default.

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ _gcp = "opentelemetry.resourcedetector.gcp_resource_detector._detector:GoogleClo
5858
[project.entry-points.opentelemetry_traces_sampler]
5959
experimental_composite_parentbased_traceidratio = "elasticotel.sdk.sampler:DefaultSampler"
6060

61+
[project.entry-points._opentelemetry_tracer_configurator]
62+
updatable_tracer_configurator = "elasticotel.sdk.trace.tracer_configurator:_updatable_tracer_configurator"
63+
6164
[project.scripts]
6265
edot-bootstrap = "elasticotel.instrumentation.bootstrap:run"
6366

‎src/elasticotel/distro/__init__.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,11 @@
4141
)
4242
from opentelemetry.sdk._configuration import _OTelSDKConfigurator
4343
from opentelemetry.sdk.environment_variables import (
44-
OTEL_METRICS_EXEMPLAR_FILTER,
4544
OTEL_EXPERIMENTAL_RESOURCE_DETECTORS,
4645
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE,
4746
OTEL_EXPORTER_OTLP_PROTOCOL,
47+
OTEL_METRICS_EXEMPLAR_FILTER,
48+
OTEL_PYTHON_TRACER_CONFIGURATOR,
4849
OTEL_TRACES_SAMPLER,
4950
OTEL_TRACES_SAMPLER_ARG,
5051
)
@@ -76,7 +77,7 @@
7677

7778
class ElasticOpenTelemetryConfigurator(_OTelSDKConfigurator):
7879
def _configure(self, **kwargs):
79-
# override GRPC and HTTP user agent headers, GRPC works since OTel SDK 1.35.0, HTTP currently requires an hack
80+
# override GRPC and HTTP user agent headers
8081
otlp_grpc_exporter_options = {
8182
"channel_options": (
8283
("grpc.primary_user_agent", f"{EDOT_GRPC_USER_AGENT_HEADER_VALUE} {_USER_AGENT_HEADER_VALUE}"),
@@ -184,6 +185,7 @@ def _configure(self, **kwargs):
184185
os.environ.setdefault(OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, "DELTA")
185186
os.environ.setdefault(OTEL_TRACES_SAMPLER, "experimental_composite_parentbased_traceidratio")
186187
os.environ.setdefault(OTEL_TRACES_SAMPLER_ARG, str(DEFAULT_SAMPLING_RATE))
188+
os.environ.setdefault(OTEL_PYTHON_TRACER_CONFIGURATOR, "updatable_tracer_configurator")
187189

188190
base_resource_detectors = [
189191
"process_runtime",

‎src/elasticotel/distro/config.py‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
from elasticotel.distro.sanitization import _sanitize_headers_env_vars
2424
from elasticotel.sdk.sampler import DefaultSampler
25+
from elasticotel.sdk.trace import tracer_configurator
2526
from opentelemetry import trace
2627
from opentelemetry._opamp import messages
2728
from opentelemetry._opamp.agent import OpAMPAgent
@@ -33,6 +34,7 @@
3334
from opentelemetry._opamp.proto import opamp_pb2 as opamp_pb2
3435
from opentelemetry.sdk._logs import LoggingHandler
3536
from opentelemetry.sdk.environment_variables import OTEL_LOG_LEVEL, OTEL_TRACES_SAMPLER_ARG
37+
from opentelemetry.sdk.trace import _TracerConfig, _TracerConfiguratorRulesT, _scope_name_matches_glob
3638

3739

3840
logger = logging.getLogger(__name__)
@@ -51,6 +53,7 @@
5153
DEFAULT_SAMPLING_RATE = 1.0
5254
DEFAULT_LOGGING_LEVEL = "warn"
5355

56+
DEACTIVATE_INSTRUMENTATIONS_CONFIG_KEY = "deactivate_instrumentations"
5457
LOGGING_LEVEL_CONFIG_KEY = "logging_level"
5558
SAMPLING_RATE_CONFIG_KEY = "sampling_rate"
5659

@@ -90,9 +93,14 @@ class Config:
9093
sampling_rate = ConfigItem(default=str(DEFAULT_SAMPLING_RATE), from_env_var=OTEL_TRACES_SAMPLER_ARG)
9194
# currently the sdk does not handle OTEL_LOG_LEVEL, so we use handle it on our own
9295
logging_level = ConfigItem(default=DEFAULT_LOGGING_LEVEL, from_env_var=OTEL_LOG_LEVEL)
96+
deactivate_instrumentations = ConfigItem(default="")
9397

9498
def to_dict(self):
95-
return {LOGGING_LEVEL_CONFIG_KEY: self.logging_level.value, SAMPLING_RATE_CONFIG_KEY: self.sampling_rate.value}
99+
return {
100+
LOGGING_LEVEL_CONFIG_KEY: self.logging_level.value,
101+
SAMPLING_RATE_CONFIG_KEY: self.sampling_rate.value,
102+
DEACTIVATE_INSTRUMENTATIONS_CONFIG_KEY: self.deactivate_instrumentations.value,
103+
}
96104

97105
def log_env_vars(self):
98106
# log all the environment variables that starts with OTEL_ or ELASTIC_OTEL_ to ease troubleshooting
@@ -142,6 +150,7 @@ def __post_init__(self):
142150
# we need to initialize each config item when we instantiate the Config and not at declaration time
143151
self.sampling_rate.init()
144152
self.logging_level.init()
153+
self.deactivate_instrumentations.init()
145154

146155
self._setup_logging()
147156

@@ -192,6 +201,42 @@ def _handle_sampling_rate(remote_config) -> ConfigUpdate:
192201
return ConfigUpdate()
193202

194203

204+
def _rules_from_deactivate_instrumentations(csv: str) -> _TracerConfiguratorRulesT:
205+
patterns = [pattern.strip() for pattern in csv.split(",") if pattern.strip()]
206+
if not patterns:
207+
return []
208+
209+
tracer_off_config = _TracerConfig(is_enabled=False)
210+
# remember that python instrumentations scope name are in the form opentelemetry.instrumentation.<module>
211+
return [(_scope_name_matches_glob(pattern), tracer_off_config) for pattern in patterns]
212+
213+
214+
def _handle_deactivate_instrumentations(remote_config) -> ConfigUpdate:
215+
tracer_provider = trace.get_tracer_provider()
216+
set_tracer_configurator = getattr(tracer_provider, "_set_tracer_configurator", None)
217+
if set_tracer_configurator is None:
218+
logger.debug("Cannot get _set_tracer_configurator from tracer provider.")
219+
return ConfigUpdate()
220+
221+
config_deactivate_instrumentations = remote_config.get(DEACTIVATE_INSTRUMENTATIONS_CONFIG_KEY, "")
222+
223+
rules = _rules_from_deactivate_instrumentations(config_deactivate_instrumentations)
224+
current_tracer_configurator = tracer_configurator._get_tracer_configurator()
225+
rules_updated = current_tracer_configurator.update_rules(rules)
226+
# if the rules did not change we are fine
227+
if not rules_updated:
228+
return ConfigUpdate()
229+
# when rules are updated we need to clear the cache of the tracer_configurator function
230+
tracer_configurator._updatable_tracer_configurator.cache_clear()
231+
232+
set_tracer_configurator(tracer_configurator=tracer_configurator._updatable_tracer_configurator)
233+
logger.debug('Updated deactivate instrumentations to "%s".', config_deactivate_instrumentations)
234+
_config = _get_config()
235+
if _config:
236+
_config.deactivate_instrumentations.update(value=config_deactivate_instrumentations)
237+
return ConfigUpdate()
238+
239+
195240
def _report_full_state(message: opamp_pb2.ServerToAgent):
196241
return message.flags & opamp_pb2.ServerToAgentFlags_ReportFullState
197242

@@ -234,6 +279,10 @@ def opamp_handler(agent: OpAMPAgent, client: OpAMPClient, message: opamp_pb2.Ser
234279
config_update = _handle_sampling_rate(remote_config)
235280
if config_update.error_message:
236281
error_messages.append(config_update.error_message)
282+
283+
config_update = _handle_deactivate_instrumentations(remote_config)
284+
if config_update.error_message:
285+
error_messages.append(config_update.error_message)
237286
except (OpAMPRemoteConfigParseException, OpAMPRemoteConfigDecodeException) as exc:
238287
logger.error(str(exc))
239288
error_messages.append(str(exc))

‎src/elasticotel/sdk/trace/__init__.py‎

Whitespace-only changes.

‎src/elasticotel/sdk/trace/tracer_configurator.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
# or more contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright
4+
# ownership. Elasticsearch B.V. licenses this file to you under
5+
# the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
from __future__ import annotations
18+
19+
import inspect
20+
from functools import lru_cache
21+
22+
from opentelemetry.sdk.trace import _TracerConfig, _TracerConfiguratorRulesT, _InstrumentationScopePredicateT
23+
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
24+
25+
26+
class _UpdatableRuleBasedTracerConfigurator:
27+
"""Updatatable version of what's available upstream"""
28+
29+
def __init__(
30+
self,
31+
*,
32+
rules: _TracerConfiguratorRulesT,
33+
default_config: _TracerConfig,
34+
):
35+
self._rules = rules
36+
self._default_config = default_config
37+
38+
def __call__(self, tracer_scope: InstrumentationScope) -> _TracerConfig:
39+
for predicate, tracer_config in list(self._rules):
40+
if predicate(tracer_scope):
41+
return tracer_config
42+
43+
# if no rule matched return the default config
44+
return self._default_config
45+
46+
@property
47+
def rules(self):
48+
return self._rules
49+
50+
def _comparable_rules(
51+
self, rules: _TracerConfiguratorRulesT
52+
) -> list[tuple[str | _InstrumentationScopePredicateT, _TracerConfig]]:
53+
"""Transform the rules to be comparable"""
54+
55+
def unpack_pattern(predicate) -> str | _InstrumentationScopePredicateT:
56+
# this assumes _scope_name_matches_glob is used to match
57+
pattern = inspect.getclosurevars(predicate).nonlocals.get("glob_pattern")
58+
if pattern is not None:
59+
return pattern
60+
return predicate
61+
62+
comparable_rules = [(unpack_pattern(predicate), config) for predicate, config in rules]
63+
return comparable_rules
64+
65+
def update_rules(self, rules: _TracerConfiguratorRulesT) -> bool:
66+
"""Updates rules if they are different than the current ones"""
67+
if self._comparable_rules(rules) == self._comparable_rules(self.rules):
68+
return False
69+
70+
self._rules = rules
71+
return True
72+
73+
74+
_tracer_configurator = _UpdatableRuleBasedTracerConfigurator(rules=[], default_config=_TracerConfig(is_enabled=True))
75+
76+
77+
def _get_tracer_configurator():
78+
global _tracer_configurator
79+
return _tracer_configurator
80+
81+
82+
@lru_cache
83+
def _updatable_tracer_configurator(
84+
tracer_scope: InstrumentationScope,
85+
) -> _TracerConfig:
86+
tracer_configurator = _get_tracer_configurator()
87+
return tracer_configurator(tracer_scope=tracer_scope)

0 commit comments

Comments
 (0)