feat: support XRootD as a default remote provider (snakemake#1017)

ryuwd · johanneskoester · web-flow · commit fe03157c3121 · 2021-08-17T17:02:17.000+02:00
* Add the remote provider XRootD to the list of allowed default remote providers.

* see what happens if we set supports_default = True for XRootD

* make sure non-relative path is supplied to xrootd

* ensure paths are always non-relative

* missing slash

* snakemake expects copying to overwrite the destination file if it already exists

* Formatting.

* Remove extraneous forward-slashes

Co-authored-by: Johannes Köster &lt;johannes.koester@uni-due.de&gt;
diff --git a/snakemake/__init__.py b/snakemake/__init__.py
@@ -1931,6 +1931,7 @@ def get_argument_parser(profile=None):
             "gridftp",
             "iRODS",
             "AzBlob",
+            "XRootD",
         ],
         help="Specify default remote provider to be used for "
         "all input and output files that don't yet specify "
diff --git a/snakemake/remote/XRootD.py b/snakemake/remote/XRootD.py
@@ -21,6 +21,8 @@
 
 
 class RemoteProvider(AbstractRemoteProvider):
+    supports_default = True
+
     def __init__(
         self, *args, keep_local=False, stay_on_remote=False, is_default=False, **kwargs
     ):
@@ -29,7 +31,7 @@ def __init__(
             keep_local=keep_local,
             stay_on_remote=stay_on_remote,
             is_default=is_default,
-            **kwargs
+            **kwargs,
         )
 
         self._xrd = XRootDHelper()
@@ -59,7 +61,7 @@ def __init__(
             keep_local=keep_local,
             stay_on_remote=stay_on_remote,
             provider=provider,
-            **kwargs
+            **kwargs,
         )
 
         if provider:
@@ -131,6 +133,12 @@ def _parse_url(self, url):
         dirname, filename = os.path.split(match.group("path"))
         # We need a trailing / to keep XRootD happy
         dirname += "/"
+
+        # and also make sure we supply a non-relative path
+        # (snakemake removes double-slash // characters)
+        if not dirname.startswith("/"):
+            dirname = "/" + dirname
+
         return domain, dirname, filename
 
     def exists(self, url):
@@ -188,18 +196,24 @@ def copy(self, source, destination):
         # Prepare the source path for XRootD
         if not self._parse_url(source):
             source = abspath(source)
+        else:
+            domain, dirname, filename = self._parse_url(source)
+            source = f"{domain}/{dirname}/{filename}"
+
         # Prepare the destination path for XRootD
         assert os.path.basename(source) == os.path.basename(destination)
         if self._parse_url(destination):
             domain, dirname, filename = self._parse_url(destination)
+            destination = f"{domain}/{dirname}/{filename}"
             self.makedirs(domain, dirname)
         else:
             destination = abspath(destination)
             if not os.path.isdir(os.path.dirname(destination)):
                 os.makedirs(os.path.dirname(destination))
+
         # Perform the copy operation
         process = client.CopyProcess()
-        process.add_job(source, destination)
+        process.add_job(source, destination, force=True)
         process.prepare()
         status, returns = process.run()
         if not status.ok or not returns[0]["status"].ok:
