Skip to content

Commit a1c39ee

Browse files
Marco-Maserajohanneskoester
Marco-Masera
and
johanneskoester
authored
fix: enhance error message formatting for strict DAG-building mode (snakemake#3376)
### Description This PR adds three CLI parameters to enhance debugging capabilities through strict evaluation modes: * strict-functions-evaluation * strict-cyclic-graph-evaluation * strict-periodic-wildcards-evaluation These flags are stored in types.DAGSettings and when enabled, enforce strict checking of exceptions that would otherwise be ignored if the rules producing them are not strictly required. The affected exceptions are: * InputFunctionException * CyclicGraphException * PeriodicWildcardError When these flags are not set, appropriate warning messages are now displayed for these exceptions. Additionally, small refactor to exceptions.py: the code to stringify exceptions is moved from print_exception into a new function named format_exception_to_string, which is now used to build consistent strings for both error reporting and logging purposes. Closes snakemake#3331 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Enabled execution of Xonsh scripts in workflows. - Consolidated strict evaluation options into a single parameter for improved flexibility. - **Documentation** - Updated help text and user guides for report generation, highlighting options for self-contained HTML and ZIP reports. - Refined deployment instructions for using conda and container integrations with workflow directives. - Added guidance on integrating Xonsh scripts within rules. - **Tests** - Expanded test coverage for conda deployments and script executions, including support for Python 3.7 and Xonsh workflows. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Johannes Köster <[email protected]>
1 parent 7755861 commit a1c39ee

File tree

5 files changed

+129
-51
lines changed

5 files changed

+129
-51
lines changed

‎snakemake/cli.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from importlib.machinery import SourceFileLoader
1111
from pathlib import Path
1212
from typing import List, Mapping, Optional, Set, Union
13-
1413
from snakemake import caching
1514
from snakemake_interface_executor_plugins.settings import ExecMode
1615
from snakemake_interface_executor_plugins.registry import ExecutorPluginRegistry
@@ -65,6 +64,7 @@
6564
SharedFSUsage,
6665
StorageSettings,
6766
WorkflowSettings,
67+
StrictDagEvaluation,
6868
)
6969
from snakemake.target_jobs import parse_target_jobs_cli_args
7070
from snakemake.utils import available_cpu_count, update_config
@@ -860,6 +860,15 @@ def get_argument_parser(profiles=None):
860860
),
861861
)
862862

863+
group_exec.add_argument(
864+
"--strict-dag-evaluation",
865+
nargs="+",
866+
choices=StrictDagEvaluation.choices(),
867+
default=set(),
868+
parse_func=StrictDagEvaluation.parse_choices_set,
869+
help="Strict evaluation of rules' correctness even when not required to produce the output files. ",
870+
)
871+
863872
try:
864873
import pulp
865874

@@ -2095,6 +2104,7 @@ def args_to_api(args, parser):
20952104
allowed_rules=args.allowed_rules,
20962105
rerun_triggers=args.rerun_triggers,
20972106
max_inventory_wait_time=args.max_inventory_time,
2107+
strict_evaluation=args.strict_dag_evaluation,
20982108
),
20992109
)
21002110

‎snakemake/dag.py‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
group_into_chunks,
3636
is_local_file,
3737
)
38-
from snakemake.settings.types import RerunTrigger
38+
from snakemake.settings.types import RerunTrigger, StrictDagEvaluation
3939
from snakemake.deployment import singularity
4040
from snakemake.exceptions import (
4141
AmbiguousRuleException,
@@ -51,6 +51,7 @@
5151
RemoteFileException,
5252
WildcardError,
5353
WorkflowError,
54+
print_exception_warning,
5455
)
5556
from snakemake.io import (
5657
_IOFile,
@@ -1045,12 +1046,26 @@ def is_strictly_higher_ordered(pivot_job):
10451046
break
10461047
except (
10471048
MissingInputException,
1048-
CyclicGraphException,
1049-
PeriodicWildcardError,
10501049
WorkflowError,
10511050
) as ex:
10521051
exceptions.append(ex)
10531052
discarded_jobs.add(job)
1053+
except (CyclicGraphException,) as ex:
1054+
if (
1055+
StrictDagEvaluation.CYCLIC_GRAPH
1056+
in self.workflow.dag_settings.strict_evaluation
1057+
):
1058+
raise ex
1059+
exceptions.append(ex)
1060+
discarded_jobs.add(job)
1061+
except (PeriodicWildcardError,) as ex:
1062+
if (
1063+
StrictDagEvaluation.PERIODIC_WILDCARDS
1064+
in self.workflow.dag_settings.strict_evaluation
1065+
):
1066+
raise ex
1067+
exceptions.append(ex)
1068+
discarded_jobs.add(job)
10541069
except RecursionError as e:
10551070
raise WorkflowError(
10561071
e,
@@ -1075,6 +1090,12 @@ def is_strictly_higher_ordered(pivot_job):
10751090
raise WorkflowError(*exceptions)
10761091
elif len(exceptions) == 1:
10771092
raise exceptions[0]
1093+
else:
1094+
for e in exceptions:
1095+
if isinstance(e, CyclicGraphException) or isinstance(
1096+
e, PeriodicWildcardError
1097+
):
1098+
print_exception_warning(e, self.workflow.linemaps)
10781099

10791100
n = len(self._dependencies)
10801101
if progress and n % 1000 == 0 and n and self._progress != n:
@@ -1180,6 +1201,10 @@ async def update_(
11801201
)
11811202
)
11821203
known_producers[res.file] = None
1204+
if isinstance(ex, CyclicGraphException) or isinstance(
1205+
ex, PeriodicWildcardError
1206+
):
1207+
print_exception_warning(ex, self.workflow.linemaps)
11831208

11841209
for file, job_ in producer.items():
11851210
dependencies[job_].add(file)
@@ -2263,10 +2288,23 @@ async def file2jobs(self, targetfile, wildcards_dict=None):
22632288
)
22642289
except InputFunctionException as e:
22652290
exceptions.append(e)
2291+
if (
2292+
StrictDagEvaluation.FUNCTIONS
2293+
in self.workflow.dag_settings.strict_evaluation
2294+
):
2295+
raise e
22662296
if not jobs:
22672297
if exceptions:
22682298
raise exceptions[0]
22692299
raise MissingRuleException(targetfile)
2300+
else:
2301+
# Warn user of possible errors
2302+
for e in exceptions:
2303+
print_exception_warning(
2304+
e,
2305+
self.workflow.linemaps,
2306+
"Use --strict-dag-evaluation to force strict mode.",
2307+
)
22702308
return jobs
22712309

22722310
def rule_dot2(self):

‎snakemake/exceptions.py‎

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -79,87 +79,106 @@ def log_verbose_traceback(ex):
7979
logger.debug(tb)
8080

8181

82-
def print_exception(ex, linemaps=None):
82+
def format_exception_to_string(ex, linemaps=None):
8383
"""
84-
Print an error message for a given exception.
84+
Returns the error message for a given exception as a string.
8585
8686
Arguments
8787
ex -- the exception
8888
linemaps -- a dict of a dict that maps for each snakefile
8989
the compiled lines to source code lines in the snakefile.
9090
"""
91-
from snakemake.logging import logger
92-
93-
log_verbose_traceback(ex)
9491
if isinstance(ex, SyntaxError) or isinstance(ex, IndentationError):
95-
logger.error(
96-
format_error(
97-
ex,
98-
ex.lineno,
99-
linemaps=linemaps,
100-
snakefile=ex.filename,
101-
show_traceback=True,
102-
)
92+
return format_error(
93+
ex,
94+
ex.lineno,
95+
linemaps=linemaps,
96+
snakefile=ex.filename,
97+
show_traceback=True,
10398
)
104-
return
99+
105100
origin = get_exception_origin(ex, linemaps) if linemaps is not None else None
106101
if origin is not None:
107102
lineno, file = origin
108-
logger.error(
109-
format_error(
110-
ex, lineno, linemaps=linemaps, snakefile=file, show_traceback=True
111-
)
103+
return format_error(
104+
ex, lineno, linemaps=linemaps, snakefile=file, show_traceback=True
112105
)
113-
return
114106
elif isinstance(ex, TokenError):
115-
logger.error(format_error(ex, None, show_traceback=False))
107+
return format_error(ex, None, show_traceback=False)
116108
elif isinstance(ex, MissingRuleException):
117-
logger.error(
118-
format_error(
119-
ex, None, linemaps=linemaps, snakefile=ex.filename, show_traceback=False
120-
)
109+
return format_error(
110+
ex, None, linemaps=linemaps, snakefile=ex.filename, show_traceback=False
121111
)
122112
elif isinstance(ex, RuleException):
113+
error_string = ""
123114
for e in ex._include:
124115
if not e.omit:
125-
logger.error(
116+
error_string += (
126117
format_error(
127118
e,
128119
e.lineno,
129120
linemaps=linemaps,
130121
snakefile=e.filename,
131122
show_traceback=True,
132123
)
124+
+ "\n"
133125
)
134-
logger.error(
135-
format_error(
136-
ex,
137-
ex.lineno,
138-
linemaps=linemaps,
139-
snakefile=ex.filename,
140-
show_traceback=True,
141-
rule=ex.rule,
142-
)
126+
error_string += format_error(
127+
ex,
128+
ex.lineno,
129+
linemaps=linemaps,
130+
snakefile=ex.filename,
131+
show_traceback=True,
132+
rule=ex.rule,
143133
)
134+
return error_string
144135
elif isinstance(ex, WorkflowError):
145-
logger.error(
146-
format_error(
147-
ex,
148-
ex.lineno,
149-
linemaps=linemaps,
150-
snakefile=ex.snakefile,
151-
show_traceback=True,
152-
rule=ex.rule,
153-
)
136+
return format_error(
137+
ex,
138+
ex.lineno,
139+
linemaps=linemaps,
140+
snakefile=ex.snakefile,
141+
show_traceback=True,
142+
rule=ex.rule,
154143
)
155144
elif isinstance(ex, ApiError):
156-
logger.error(f"Error: {ex}")
145+
return f"Error: {ex}"
157146
elif isinstance(ex, CliException):
158-
logger.error(f"Error: {ex}")
147+
return f"Error: {ex}"
159148
elif isinstance(ex, KeyboardInterrupt):
160-
logger.info("Cancelling snakemake on user request.")
149+
return "Cancelling snakemake on user request."
161150
else:
162-
logger.error("\n".join(traceback.format_exception(ex)))
151+
return "\n".join(traceback.format_exception(ex))
152+
153+
154+
def print_exception_warning(ex, linemaps=None, footer_message=""):
155+
"""
156+
Print an error message for a given exception using logger warning.
157+
158+
Arguments
159+
ex -- the exception
160+
linemaps -- a dict of a dict that maps for each snakefile
161+
the compiled lines to source code lines in the snakefile.
162+
"""
163+
from snakemake.logging import logger
164+
165+
log_verbose_traceback(ex)
166+
logger.warning(f"{format_exception_to_string(ex, linemaps)}\n{footer_message}")
167+
168+
169+
def print_exception(ex, linemaps=None):
170+
"""
171+
Print an error message for a given exception.
172+
173+
Arguments
174+
ex -- the exception
175+
linemaps -- a dict of a dict that maps for each snakefile
176+
the compiled lines to source code lines in the snakefile.
177+
"""
178+
from snakemake.logging import logger
179+
180+
log_verbose_traceback(ex)
181+
logger.error(format_exception_to_string(ex, linemaps))
163182

164183

165184
def update_lineno(ex: SyntaxError, linemaps):

‎snakemake/settings/enums.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,9 @@ class Quietness(SettingsEnumBase):
2626
PROGRESS = 1
2727
ALL = 2
2828
HOST = 3
29+
30+
31+
class StrictDagEvaluation(SettingsEnumBase):
32+
FUNCTIONS = 0
33+
CYCLIC_GRAPH = 1
34+
PERIODIC_WILDCARDS = 2

‎snakemake/settings/types.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
ChangeType,
3535
CondaCleanupPkgs,
3636
Quietness,
37+
StrictDagEvaluation,
3738
)
3839

3940

@@ -202,6 +203,10 @@ class DAGSettings(SettingsBase):
202203
allowed_rules: AnySet[str] = frozenset()
203204
rerun_triggers: AnySet[RerunTrigger] = RerunTrigger.all()
204205
max_inventory_wait_time: int = 20
206+
strict_evaluation: AnySet[StrictDagEvaluation] = frozenset()
207+
# strict_functions_evaluation: bool = False
208+
# strict_cycle_evaluation: bool = False
209+
# strict_wildcards_recursion_evaluation: bool = False
205210

206211
def _check(self):
207212
if self.batch is not None and self.forceall:

0 commit comments

Comments
 (0)