Merge branch 'main' into fix/resource_parsing

cmeesters · web-flow · commit 100396dbbb4a · 2025-04-02T07:53:01.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,22 @@
 # Changelog
 
 
+## [9.1.4](https://github.com/snakemake/snakemake/compare/v9.1.3...v9.1.4) (2025-04-01)
+
+
+### Bug Fixes
+
+* Fix map call in report creation ([#3503](https://github.com/snakemake/snakemake/issues/3503)) ([44754cc](https://github.com/snakemake/snakemake/commit/44754ccbafaf94d30b9e507b81e8fb5f38409e3d))
+* in the report, do not render toggle labels if there is more than one label eligible for a toggle ([#3502](https://github.com/snakemake/snakemake/issues/3502)) ([3be8ca9](https://github.com/snakemake/snakemake/commit/3be8ca9308df72494cbca4d6f572486c8486418a))
+* only inform about storage cleanup in case of --verbose mode ([#3494](https://github.com/snakemake/snakemake/issues/3494)) ([62bbbf5](https://github.com/snakemake/snakemake/commit/62bbbf5129b2fe9ed125245fd90b69d53db8b6ce))
+* provide causing rule to workflow error during checkpoint handling ([#3499](https://github.com/snakemake/snakemake/issues/3499)) ([b4cbe5d](https://github.com/snakemake/snakemake/commit/b4cbe5d2e9d5d985466a5dac4c985caa66efb5ad))
+* rerun-trigger regression ([#3492](https://github.com/snakemake/snakemake/issues/3492)) ([811742d](https://github.com/snakemake/snakemake/commit/811742d5db425219a3018f04190a99f53b3b742d))
+
+
+### Documentation
+
+* note about parameter inheritance when using the "use rule with" pattern ([#3500](https://github.com/snakemake/snakemake/issues/3500)) ([8f6a8eb](https://github.com/snakemake/snakemake/commit/8f6a8eb783bf003dbf804df737a2adfbfa9818ef))
+
 ## [9.1.3](https://github.com/snakemake/snakemake/compare/v9.1.2...v9.1.3) (2025-03-26)
 
 
diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst
@@ -2931,6 +2931,12 @@ In reality, one will often change more.
 Analogously to the ``use rule`` from external modules, any properties of the rule (``input``, ``output``, ``log``, ``params``, ``benchmark``, ``threads``, ``resources``, etc.) can be modified, except the actual execution step (``shell``, ``notebook``, ``script``, ``cwl``, or ``run``).
 All unmodified properties are inherited from the parent rule.
 
+.. note::
+    Modification of `params` allows the replacement of single keyword arguments.
+    Keyword `params` arguments of the original rule that are not defined after `with` are inherited.
+    Positional `params` arguments of the original rule are overwritten, if positional `params` arguments are given after `with`.
+    All other properties (``input``, ``output``, ...) are entirely overwritten with the values specified after `with`.
+
 .. _snakefiles-aux_source_files:
 
 Accessing auxiliary source files
diff --git a/src/snakemake/cli.py b/src/snakemake/cli.py
@@ -721,7 +721,7 @@ def get_argument_parser(profiles=None):
         "--rerun-triggers",
         nargs="+",
         choices=RerunTrigger.choices(),
-        default=RerunTrigger.choices(),
+        default=RerunTrigger.all(),
         parse_func=RerunTrigger.parse_choices_set,
         help="Define what triggers the rerunning of a job. By default, "
         "all triggers are used, which guarantees that results are "
diff --git a/src/snakemake/dag.py b/src/snakemake/dag.py
@@ -506,7 +506,7 @@ async def cleanup_storage_objects(self):
                             and f not in cleaned
                             and not f.should_keep_local
                         ):
-                            logger.info(
+                            logger.debug(
                                 "Cleaning up local remainders of "
                                 f"{f.storage_object.print_query}"
                             )
diff --git a/src/snakemake/report/__init__.py b/src/snakemake/report/__init__.py
@@ -3,6 +3,7 @@
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
+from asyncio import TaskGroup
 from dataclasses import InitVar, dataclass, field
 import os
 import sys
@@ -13,12 +14,9 @@
 import io
 from typing import List, Optional
 import uuid
-import json
-import time
 import itertools
 from collections import defaultdict
 import hashlib
-from zipfile import ZipFile, ZIP_DEFLATED
 from pathlib import Path
 import numbers
 
@@ -113,7 +111,7 @@ def report(
     if outmime != "text/html":
         raise ValueError("Path to report output has to be an HTML file.")
     definitions = textwrap.dedent(
-        """
+        """[]
     .. role:: raw-html(raw)
        :format: html
 
@@ -187,11 +185,21 @@ def report(
     )
 
 
-def expand_report_argument(item, wildcards, job):
+async def expand_report_argument(item, wildcards, job):
     if is_callable(item):
         aux_params = get_input_function_aux_params(
             item, {"params": job.params, "input": job.input, "output": job.output}
         )
+        io_items = ["input", "output"]
+        if any(io_item in aux_params for io_item in io_items):
+            # retrieve all input or output files from storage before evaluating function
+            async with TaskGroup() as tg:
+                for io_item in io_items:
+                    if io_item in aux_params:
+                        for f in aux_params[io_item]:
+                            if f.is_storage:
+                                tg.create_task(f.retrieve_from_storage())
+
         try:
             item = item(wildcards, **aux_params)
         except Exception as e:
@@ -209,15 +217,11 @@ def expand_report_argument(item, wildcards, job):
 
 @dataclass(slots=True)
 class Category(CategoryInterface):
-    wildcards: InitVar
-    job: InitVar
     name: Optional[str]
     is_other: bool = field(init=False)
     id: str = field(init=False)
 
-    def __post_init__(self, wildcards, job):
-        if self.name is not None:
-            self.name = expand_report_argument(self.name, wildcards, job)
+    def __post_init__(self):
         if self.name is None:
             self.name = "Other"
 
@@ -447,10 +451,10 @@ def workflow(self):
         return self.job.rule.workflow
 
 
-def expand_labels(labels, wildcards, job):
+async def expand_labels(labels, wildcards, job):
     if labels is None:
         return None
-    labels = expand_report_argument(labels, wildcards, job)
+    labels = await expand_report_argument(labels, wildcards, job)
 
     if labels is None:
         return None
@@ -469,7 +473,7 @@ def expand_labels(labels, wildcards, job):
             rule=job.rule,
         )
     return {
-        name: expand_report_argument(col, wildcards, job)
+        name: await expand_report_argument(col, wildcards, job)
         for name, col in labels.items()
     }
 
@@ -541,21 +545,32 @@ def get_time(rectime, metatime, sel_func):
                     )
                 report_obj = get_flag_value(f, "report")
 
-                def register_file(
+                async def register_file(
                     f,
                     parent_path=None,
                     wildcards_overwrite=None,
                     aux_files=None,
                     name_overwrite=None,
                 ):
                     wildcards = wildcards_overwrite or job.wildcards
+
+                    async def expand_cat_name(cat_name, wildcards, job):
+                        if cat_name is not None:
+                            return await expand_report_argument(
+                                cat_name, wildcards, job
+                            )
+                        else:
+                            return cat_name
+
                     category = Category(
-                        name=report_obj.category, wildcards=wildcards, job=job
+                        name=await expand_cat_name(report_obj.category, wildcards, job),
                     )
                     subcategory = Category(
-                        name=report_obj.subcategory, wildcards=wildcards, job=job
+                        name=await expand_cat_name(
+                            report_obj.subcategory, wildcards, job
+                        ),
                     )
-                    labels = expand_labels(report_obj.labels, wildcards, job)
+                    labels = await expand_labels(report_obj.labels, wildcards, job)
 
                     results[category][subcategory].append(
                         FileRecord(
@@ -577,7 +592,7 @@ def register_file(
                 if f.is_storage:
                     await f.retrieve_from_storage()
                 if os.path.isfile(f):
-                    register_file(f)
+                    await register_file(f)
                 elif os.path.isdir(f):
                     if report_obj.htmlindex:
                         aux_files = []
@@ -598,7 +613,7 @@ def register_file(
                                 "Given htmlindex {} not found in directory "
                                 "marked for report".format(report_obj.htmlindex)
                             )
-                        register_file(
+                        await register_file(
                             os.path.join(f, report_obj.htmlindex),
                             aux_files=aux_files,
                             name_overwrite=f"{os.path.basename(f)}.html",
@@ -621,7 +636,7 @@ def register_file(
                                 w.update(job.wildcards_dict)
                                 w = Wildcards(fromdict=w)
                                 subfile = apply_wildcards(pattern, w)
-                                register_file(
+                                await register_file(
                                     subfile, parent_path=f, wildcards_overwrite=w
                                 )
                         if not found_something:
diff --git a/src/snakemake/report/html_reporter/template/components/abstract_results.js b/src/snakemake/report/html_reporter/template/components/abstract_results.js
@@ -98,7 +98,7 @@ class AbstractResults extends React.Component {
             return { data: data, toggles };
         }, function () {
             if (_this.state.data.resultPathsToEntryLabels.has(_this.props.app.state.resultPath)) {
-                let toggleLabels = Array.from(data.toggleLabels.keys().map((label) => _this.state.toggles.get(label)));
+                let toggleLabels = Array.from(data.toggleLabels.keys()).map((label) => _this.state.toggles.get(label));
                 let entryLabels = _this.state.data.resultPathsToEntryLabels.get(_this.props.app.state.resultPath);
                 let targetPath = _this.state.data.entries.get(arrayKey(entryLabels)).get(arrayKey(toggleLabels));
                 _this.toggleViewManager.handleSelectedResult(targetPath);
@@ -190,6 +190,10 @@ class AbstractResults extends React.Component {
                 }
             });
         }
+        // Only allow one toggle label for now, in order to avoid confusion in the UI
+        if (toggleLabels.size > 1) {
+            toggleLabels = new Map();
+        }
 
         let entries = new Map();
         let entryLabelValues = [];
@@ -318,4 +322,4 @@ class AbstractResults extends React.Component {
             ];
         });
     }
-}
+}
diff --git a/src/snakemake/rules.py b/src/snakemake/rules.py
@@ -937,8 +937,12 @@ def handle_incomplete_checkpoint(exception):
                 return TBDString()
             else:
                 raise WorkflowError(
-                    "Rule parameter depends on checkpoint but checkpoint output is not defined as input file for the rule. "
-                    "Please add the output of the respective checkpoint to the rule inputs."
+                    "Rule parameter depends on checkpoint but checkpoint output is not "
+                    "defined as input file for the rule. Please add the output of the "
+                    "respective checkpoint to the rule inputs. "
+                    f"Input: {','.join(input)} "
+                    f"Checkpoint file: {exception.targetfile}",
+                    rule=self,
                 )
 
         # We make sure that resources are only evaluated if a param function

Original file line number	Diff line number	Diff line change
`@@ -506,7 +506,7 @@ async def cleanup_storage_objects(self):`
`506`	`506`	`and f not in cleaned`
`507`	`507`	`and not f.should_keep_local`
`508`	`508`	`):`
`509`		`- logger.info(`
	`509`	`+ logger.debug(`
`510`	`510`	`"Cleaning up local remainders of "`
`511`	`511`	`f"{f.storage_object.print_query}"`
`512`	`512`	`)`