Stop checking for unneeded temp files from groups

pvandyken · pvandyken · commit f73b3cf887c0 · 2023-01-25T20:33:54.000-05:00
Files produced by jobs in the middle of a group job that are neither
endpoints nor needed by any other jobs in the dag are no longer subject
to an existance check
diff --git a/snakemake/dag.py b/snakemake/dag.py
@@ -579,6 +579,12 @@ def check_and_touch_output(
         if job.benchmark:
             expanded_output.append(job.benchmark)
 
+        if isinstance(ignore_missing_output, list):
+            expanded_output = [
+                f for f in expanded_output if f not in ignore_missing_output
+            ]
+            ignore_missing_output = []
+
         if not ignore_missing_output:
             try:
                 wait_for_files(
diff --git a/snakemake/jobs.py b/snakemake/jobs.py
@@ -1461,6 +1461,21 @@ def cleanup(self):
             job.cleanup()
 
     def postprocess(self, error=False, **kwargs):
+        def needed(job_, f):
+            return any(
+                f in files
+                for j, files in self.dag.depending[job_].items()
+                if (
+                    not self.dag.finished(j)
+                    and self.dag.needrun(j)
+                    and j not in self.jobs
+                )
+            ) or f in self.dag.targetfiles
+
+        kwargs["ignore_missing_output"] = [
+            f for j in self.jobs for f in j.output
+            if is_flagged(f, "temp") and not needed(j, f)
+        ]
         # Iterate over jobs in toposorted order (see self.__iter__) to
         # ensure that outputs are touched in correct order.
         for level in self.toposorted:
