Merge branch 'main' into fix/issue3001

cmeesters · web-flow · commit 12ad41d2a915 · 2024-08-12T17:17:42.000+02:00
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
@@ -15,3 +15,7 @@
 #content {
     overflow-x: auto;
 }
+
+code {
+    font-family: monospace !important;
+}
diff --git a/docs/project_info/faq.rst b/docs/project_info/faq.rst
@@ -356,30 +356,11 @@ On unix, you can make use of the commonly pre-installed `mail` command:
 
 In case your administrator does not provide you with a proper configuration of the sendmail framework, you can configure `mail` to work e.g. via Gmail (see `here <https://www.cyberciti.biz/tips/linux-use-gmail-as-a-smarthost.html>`_).
 
-I want to pass variables between rules. Is that possible?
----------------------------------------------------------
-
-Because of the cluster support and the ability to resume a workflow where you stopped last time, Snakemake in general should be used in a way that information is stored in the output files of your jobs. Sometimes it might though be handy to have a kind of persistent storage for simple values between jobs and rules. Using plain python objects like a global dict for this will not work as each job is run in a separate process by snakemake. What helps here is the `PersistentDict` from the `pytools <https://github.com/inducer/pytools>`_ package. Here is an example of a Snakemake workflow using this facility:
-
-.. code-block:: python
-
-    from pytools.persistent_dict import PersistentDict
-
-    storage = PersistentDict("mystorage")
-
-    rule a:
-        input: "test.in"
-        output: "test.out"
-        run:
-            myvar = storage.fetch("myvar")
-            # do stuff
-
-    rule b:
-        output: temp("test.in")
-        run:
-            storage.store("myvar", 3.14)
+I want to pass Python variables between rules. Is that possible?
+----------------------------------------------------------------
 
-Here, the output rule b has to be temp in order to ensure that ``myvar`` is stored in each run of the workflow as rule a relies on it. In other words, the `PersistentDict` is persistent between the job processes, but not between different runs of this workflow. If you need to conserve information between different runs, use output files for them.
+Because of the cluster support and the ability to resume a workflow where you stopped last time, Snakemake in general should be used in a way that information is stored in the output files of your jobs.
+A common approach to pass non file variable data between rules is to use json or parquet for writing in the one rule and reading in a consuming rule the variable shall be passed to.
 
 Why do my global variables behave strangely when I run my job on a cluster?
 ---------------------------------------------------------------------------
diff --git a/snakemake/dag.py b/snakemake/dag.py
@@ -437,10 +437,10 @@ async def cleanup_storage_objects(self):
     async def sanitize_local_storage_copies(self):
         """Remove local copies of storage files that will be recreated in this run."""
         async with asyncio.TaskGroup() as tg:
-            async for job in self.needrun_jobs:
+            for job in self.needrun_jobs():
                 if not self.finished(job):
                     for f in job.output:
-                        if f.is_storage and f.exists_local():
+                        if f.is_storage and await f.exists_local():
                             tg.create_task(
                                 f.remove(remove_non_empty_dir=True, only_local=True)
                             )
@@ -1609,7 +1609,7 @@ async def postprocess(
             # cleanup local storage copies of files that will be created by jobs
             # this is important to ensure that there are no outdated local copies
             # that misguide e.g. params functions.
-            self.sanitize_local_storage_copies()
+            await self.sanitize_local_storage_copies()
             self.update_container_imgs()
             self.update_conda_envs()
             await self.update_needrun()
diff --git a/test-environment.yml b/test-environment.yml
@@ -59,6 +59,7 @@ dependencies:
   - nodejs # for cwltool
   - immutables
   - conda
+  - paramiko >=3.4.1 # TODO remove line (paramiko is an indirect dependency), just a temporal workaround for this bug: https://github.com/paramiko/paramiko/issues/2419
   - pip
   - pip:
       - cwltool
diff --git a/tests/test_paramspace/expected-results/results/default/simulations/alpha~1/beta~0.1/gamma~0.99.tsv b/tests/test_paramspace/expected-results/results/default/simulations/alpha~1/beta~0.1/gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/default/simulations/alpha~2/beta~0.0/gamma~3.9.tsv b/tests/test_paramspace/expected-results/results/default/simulations/alpha~2/beta~0.0/gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/expected-results/results/empty/simulations/alpha~1/beta~0.1/gamma~0.99.tsv b/tests/test_paramspace/expected-results/results/empty/simulations/alpha~1/beta~0.1/gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/empty/simulations/alpha~2/beta~0.0/gamma~3.9.tsv b/tests/test_paramspace/expected-results/results/empty/simulations/alpha~2/beta~0.0/gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/expected-results/results/filenamesep/simulations/alpha~1__beta~0.1__gamma~0.99.tsv b/tests/test_paramspace/expected-results/results/filenamesep/simulations/alpha~1__beta~0.1__gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/filenamesep/simulations/alpha~2__beta~0.0__gamma~3.9.tsv b/tests/test_paramspace/expected-results/results/filenamesep/simulations/alpha~2__beta~0.0__gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/expected-results/results/full/simulations/alpha~1_beta~0.1_gamma~0.99.tsv b/tests/test_paramspace/expected-results/results/full/simulations/alpha~1_beta~0.1_gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/full/simulations/alpha~2_beta~0.0_gamma~3.9.tsv b/tests/test_paramspace/expected-results/results/full/simulations/alpha~2_beta~0.0_gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/expected-results/results/full_reorder/simulations/beta~0.0_gamma~3.9_alpha~2.tsv b/tests/test_paramspace/expected-results/results/full_reorder/simulations/beta~0.0_gamma~3.9_alpha~2.tsv
@@ -1 +1 @@
-{'beta': 0.0, 'gamma': 3.9, 'alpha': 2}
+{"beta": 0.0, "gamma": 3.9, "alpha": 2}
diff --git a/tests/test_paramspace/expected-results/results/full_reorder/simulations/beta~0.1_gamma~0.99_alpha~1.tsv b/tests/test_paramspace/expected-results/results/full_reorder/simulations/beta~0.1_gamma~0.99_alpha~1.tsv
@@ -1 +1 @@
-{'beta': 0.1, 'gamma': 0.99, 'alpha': 1}
+{"beta": 0.1, "gamma": 0.99, "alpha": 1}
diff --git a/tests/test_paramspace/expected-results/results/one/simulations/alpha~1/gamma~0.99/beta~0.1.tsv b/tests/test_paramspace/expected-results/results/one/simulations/alpha~1/gamma~0.99/beta~0.1.tsv
@@ -1 +1 @@
-{'alpha': 1, 'gamma': 0.99, 'beta': 0.1}
+{"alpha": 1, "gamma": 0.99, "beta": 0.1}
diff --git a/tests/test_paramspace/expected-results/results/one/simulations/alpha~2/gamma~3.9/beta~0.0.tsv b/tests/test_paramspace/expected-results/results/one/simulations/alpha~2/gamma~3.9/beta~0.0.tsv
@@ -1 +1 @@
-{'alpha': 2, 'gamma': 3.9, 'beta': 0.0}
+{"alpha": 2, "gamma": 3.9, "beta": 0.0}
diff --git a/tests/test_paramspace/expected-results/results/sep/simulations/alpha_is_1/beta_is_0.1/gamma_is_0.99.tsv b/tests/test_paramspace/expected-results/results/sep/simulations/alpha_is_1/beta_is_0.1/gamma_is_0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/sep/simulations/alpha_is_2/beta_is_0.0/gamma_is_3.9.tsv b/tests/test_paramspace/expected-results/results/sep/simulations/alpha_is_2/beta_is_0.0/gamma_is_3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/expected-results/results/sep_and_pattern/simulations/beta=0.0_gamma=3.9_alpha=2.tsv b/tests/test_paramspace/expected-results/results/sep_and_pattern/simulations/beta=0.0_gamma=3.9_alpha=2.tsv
@@ -1 +1 @@
-{'beta': 0.0, 'gamma': 3.9, 'alpha': 2}
+{"beta": 0.0, "gamma": 3.9, "alpha": 2}
diff --git a/tests/test_paramspace/expected-results/results/sep_and_pattern/simulations/beta=0.1_gamma=0.99_alpha=1.tsv b/tests/test_paramspace/expected-results/results/sep_and_pattern/simulations/beta=0.1_gamma=0.99_alpha=1.tsv
@@ -1 +1 @@
-{'beta': 0.1, 'gamma': 0.99, 'alpha': 1}
+{"beta": 0.1, "gamma": 0.99, "alpha": 1}
diff --git a/tests/test_paramspace/expected-results/results/two/simulations/alpha~1/beta~0.1_gamma~0.99.tsv b/tests/test_paramspace/expected-results/results/two/simulations/alpha~1/beta~0.1_gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace/expected-results/results/two/simulations/alpha~2/beta~0.0_gamma~3.9.tsv b/tests/test_paramspace/expected-results/results/two/simulations/alpha~2/beta~0.0_gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace/scripts/simulate.py b/tests/test_paramspace/scripts/simulate.py
@@ -1,2 +1,5 @@
+import json
+
 with open(snakemake.output[0], "w") as out:
-    print(snakemake.params.simulation, file=out)
+    # convert numpy types to native python types for serialization
+    json.dump({k: v.item() for k, v in snakemake.params.simulation.items()}, out)
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/default/simulations/alpha~1/beta~0.1/gamma~0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/default/simulations/alpha~1/beta~0.1/gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/default/simulations/alpha~2/beta~0.0/gamma~3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/default/simulations/alpha~2/beta~0.0/gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/empty/simulations/alpha~1/beta~0.1/gamma~0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/empty/simulations/alpha~1/beta~0.1/gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/empty/simulations/alpha~2/beta~0.0/gamma~3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/empty/simulations/alpha~2/beta~0.0/gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/filenamesep/simulations/alpha~1__beta~0.1__gamma~0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/filenamesep/simulations/alpha~1__beta~0.1__gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/filenamesep/simulations/alpha~2__beta~0.0__gamma~3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/filenamesep/simulations/alpha~2__beta~0.0__gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/full/simulations/alpha~1_beta~0.1_gamma~0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/full/simulations/alpha~1_beta~0.1_gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/full/simulations/alpha~2_beta~0.0_gamma~3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/full/simulations/alpha~2_beta~0.0_gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/full_reorder/simulations/beta~0.0_gamma~3.9_alpha~2.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/full_reorder/simulations/beta~0.0_gamma~3.9_alpha~2.tsv
@@ -1 +1 @@
-{'beta': 0.0, 'gamma': 3.9, 'alpha': 2}
+{"beta": 0.0, "gamma": 3.9, "alpha": 2}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/full_reorder/simulations/beta~0.1_gamma~0.99_alpha~1.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/full_reorder/simulations/beta~0.1_gamma~0.99_alpha~1.tsv
@@ -1 +1 @@
-{'beta': 0.1, 'gamma': 0.99, 'alpha': 1}
+{"beta": 0.1, "gamma": 0.99, "alpha": 1}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/one/simulations/alpha~1/gamma~0.99/beta~0.1.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/one/simulations/alpha~1/gamma~0.99/beta~0.1.tsv
@@ -1 +1 @@
-{'alpha': 1, 'gamma': 0.99, 'beta': 0.1}
+{"alpha": 1, "gamma": 0.99, "beta": 0.1}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/one/simulations/alpha~2/gamma~3.9/beta~0.0.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/one/simulations/alpha~2/gamma~3.9/beta~0.0.tsv
@@ -1 +1 @@
-{'alpha': 2, 'gamma': 3.9, 'beta': 0.0}
+{"alpha": 2, "gamma": 3.9, "beta": 0.0}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/sep/simulations/alpha_is_1/beta_is_0.1/gamma_is_0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/sep/simulations/alpha_is_1/beta_is_0.1/gamma_is_0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/sep/simulations/alpha_is_2/beta_is_0.0/gamma_is_3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/sep/simulations/alpha_is_2/beta_is_0.0/gamma_is_3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/sep_and_pattern/simulations/beta=0.0_gamma=3.9_alpha=2.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/sep_and_pattern/simulations/beta=0.0_gamma=3.9_alpha=2.tsv
@@ -1 +1 @@
-{'beta': 0.0, 'gamma': 3.9, 'alpha': 2}
+{"beta": 0.0, "gamma": 3.9, "alpha": 2}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/sep_and_pattern/simulations/beta=0.1_gamma=0.99_alpha=1.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/sep_and_pattern/simulations/beta=0.1_gamma=0.99_alpha=1.tsv
@@ -1 +1 @@
-{'beta': 0.1, 'gamma': 0.99, 'alpha': 1}
+{"beta": 0.1, "gamma": 0.99, "alpha": 1}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/two/simulations/alpha~1/beta~0.1_gamma~0.99.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/two/simulations/alpha~1/beta~0.1_gamma~0.99.tsv
@@ -1 +1 @@
-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}
+{"alpha": 1, "beta": 0.1, "gamma": 0.99}
diff --git a/tests/test_paramspace_single_wildcard/expected-results/results/two/simulations/alpha~2/beta~0.0_gamma~3.9.tsv b/tests/test_paramspace_single_wildcard/expected-results/results/two/simulations/alpha~2/beta~0.0_gamma~3.9.tsv
@@ -1 +1 @@
-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}
+{"alpha": 2, "beta": 0.0, "gamma": 3.9}
diff --git a/tests/test_paramspace_single_wildcard/scripts/simulate.py b/tests/test_paramspace_single_wildcard/scripts/simulate.py
@@ -1,2 +1,5 @@
+import json
+
 with open(snakemake.output[0], "w") as out:
-    print(snakemake.params.simulation, file=out)
+    # convert numpy types to native python types for serialization
+    json.dump({k: v.item() for k, v in snakemake.params.simulation.items()}, out)
diff --git a/tests/test_persistent_dict/Snakefile b/tests/test_persistent_dict/Snakefile
diff --git a/tests/test_persistent_dict/expected-results/.gitignore b/tests/test_persistent_dict/expected-results/.gitignore
diff --git a/tests/tests.py b/tests/tests.py
@@ -304,15 +304,6 @@ def test_ruledeps():
     run(dpath("test_ruledeps"))
 
 
-def test_persistent_dict():
-    try:
-        import pytools
-
-        run(dpath("test_persistent_dict"))
-    except ImportError:
-        pass
-
-
 @connected
 def test_url_include():
     run(dpath("test_url_include"))

Original file line number	Diff line number	Diff line change
`@@ -15,3 +15,7 @@`
`15`	`15`	`#content {`
`16`	`16`	`overflow-x: auto;`
`17`	`17`	`}`
	`18`	`+`
	`19`	`+code {`
	`20`	`+ font-family: monospace !important;`
	`21`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-{'alpha': 1, 'beta': 0.1, 'gamma': 0.99}`
	`1`	`+{"alpha": 1, "beta": 0.1, "gamma": 0.99}`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-{'alpha': 2, 'beta': 0.0, 'gamma': 3.9}`
	`1`	`+{"alpha": 2, "beta": 0.0, "gamma": 3.9}`