perf: compare checksums of input files < 1MB (before (10KB); move source cache into .snakemake directory in the workdir instead of the user specific app cache (improves multi-user capability of workflows)

johanneskoester · johanneskoester · commit faf5c1bd84f3 · 2025-01-23T12:18:14.000+01:00
diff --git a/snakemake/io.py b/snakemake/io.py
@@ -600,15 +600,15 @@ async def is_checksum_eligible(self):
         return (
             await self.exists_local()
             and not os.path.isdir(self.file)
-            and await self.size() < 100000
+            and await self.size() <= 1000000
             and not self.is_fifo()
         )
 
     async def checksum(self, force=False):
         """Return checksum if file is small enough, else None.
         Returns None if file does not exist. If force is True,
         omit eligibility check."""
-        if force or await self.is_checksum_eligible():  # less than 100000 bytes
+        if force or await self.is_checksum_eligible():  # less than 1 MB
             checksum = sha256()
             if await self.size() > 0:
                 # only read if file is bigger than zero
diff --git a/snakemake/sourcecache.py b/snakemake/sourcecache.py
@@ -9,6 +9,7 @@
 import os
 import shutil
 import stat
+from typing import Optional
 from snakemake import utils
 import tempfile
 import io
@@ -231,6 +232,12 @@ def __init__(
     def __post_init__(self):
         pass
 
+    def mtime(self) -> Optional[float]:
+        # Intentionally None, hence causing any caching to generate an updated mtime.
+        # Switching commits/branches/refs in the same repo should cause rerun triggers
+        # if those files are used as input files for jobs and have changed checksums.
+        return None
+
     def is_persistently_cacheable(self):
         return bool(self.tag or self.commit)
 
diff --git a/snakemake/workflow.py b/snakemake/workflow.py
@@ -252,11 +252,15 @@ def register_resource(self, name: str, value: Union[int, str]):
     def source_cache_path(self) -> Path:
         assert self.storage_settings is not None
         if SharedFSUsage.SOURCE_CACHE not in self.storage_settings.shared_fs_usage:
+            # TODO can this cause issues with the source cache?
+            # When regenerating it for each remote job, might that accidentally trigger
+            # job reruns where an input file or a source file suddenly becomes newer?
             return self.snakemake_tmp_dir / "source-cache"
         else:
-            return Path(
-                os.path.join(get_appdirs().user_cache_dir, "snakemake/source-cache")
-            )
+            # The source cache should be shared across different users of the same
+            # workflow. Hence we have to make sure that it is in a directory that is
+            # the same for all people that run the same workflow in the same workdir.
+            return self.persistence.path / "source-cache"
 
     @property
     def storage_registry(self):
