Merge pull request #613 from jkloetzke/url-improvements

Url SCM improvements

Author: jkloetzke

doc/manual/configuration.rst

@@ -1152,17 +1152,22 @@ url
    is performed. This replaces the initial ``~`` or ``~user`` by the *user*’s
    home directory.
 
-   If a SHA digest is
-   given with ``digestSHA1``, ``digestSHA256`` and/or ``digestSHA512``, the
-   downloaded file will be checked for a matching hash sum. This also makes the
-   URL deterministic for Bob. Otherwise the URL will be checked in each build
-   for updates. Based on the file name ending, Bob will try to extract the
-   downloaded file. You may prevent this by setting the ``extract`` attribute
-   to ``no`` or ``False``. If the heuristic fails, the extraction tool may be
-   specified as ``tar``, ``gzip``, ``xz``, ``7z`` or ``zip`` directly. For
-   ``tar`` files it is possible to strip a configurable number of leading
-   components from file names on extraction by the ``stripComponents``
-   attribute.
+   If a SHA digest is given with ``digestSHA1``, ``digestSHA256`` and/or
+   ``digestSHA512``, the downloaded file will be checked for a matching hash
+   sum. This also makes the URL deterministic for Bob. Otherwise, the URL will
+   be checked in each build for updates.
+
+   Based on the file name ending, Bob will try to extract the downloaded file.
+   You may prevent this by setting the ``extract`` attribute to ``no`` or
+   ``False``. If the heuristic fails, the extraction tool may be specified as
+   ``tar``, ``gzip``, ``xz``, ``7z`` or ``zip`` directly. For ``tar`` files it
+   is possible to strip a configurable number of leading components from file
+   names on extraction by the ``stripComponents`` attribute.
+
+   If the file is extracted, the original file will be kept next to the
+   ``workspace`` to keep the directory clean. If the
+   :ref:`policies-urlScmSeparateDownload` policy is on the old behavour, the
+   downloaded file will still be in put the workspace, though.
 
    .. note::
        Starting with Bob 0.14 (see :ref:`policies-tidyUrlScm` policy) the whole

doc/manual/policies.rst

@@ -344,13 +344,15 @@ New behavior
 
     Unmanaged layers are expected in the same directory.
 
+.. _policies-urlScmSeparateDownload:
+
 urlScmSeparateDownload
 ~~~~~~~~~~~~~~~~~~~~~~
 
 Introduced in: 1.0
 
-This policy controls where bob places downloaded files of UrlScms if extraction is
-used.
+This policy controls where Bob places downloaded files of ``url`` SCMs if
+extraction is used.
 
 Old behavior
     The downloaded file could be found in the workspace next to the extracted files.

pym/bob/invoker.py

@@ -192,21 +192,27 @@ async def __runCommand(self, args, cwd, stdout=None, stderr=None,
         elif stdout == False:
             stdoutRedir = subprocess.DEVNULL
             stdoutStreams = []
-        else:
+        elif stdout is None:
             stdoutRedir = self.__stdout
             stdoutStreams = [self.__stdoutStream]
+        else:
+            stdoutRedir = stdout
+            stdoutStreams = []
 
         if stderr == True:
             # If stderr should be captured we use a dedicated buffer. This
             # buffer is then returned to the caller at child exit.
             stderrRedir = subprocess.PIPE
             stderrStreams = [io.BytesIO(), self.__stderrStream]
-        elif stdout == False:
+        elif stderr == False:
             stderrRedir = subprocess.DEVNULL
             stderrStreams = []
-        else:
+        elif stderr is None:
             stderrRedir = self.__stderr
             stderrStreams = [self.__stderrStream]
+        else:
+            stderrRedir = stderr
+            stderrStreams = []
 
         # Sanity check on Windows that there are no environment variables that
         # differ only in case. The Windows envrionment is used case insensitive

pym/bob/scm/url.py

@@ -158,19 +158,23 @@ def dumpMode(mode):
 
 
 class Extractor():
+    SUPPORT_STRIP = False
+
     def __init__(self, dir, file, strip, separateDownload):
+        if strip != 0 and not self.SUPPORT_STRIP:
+            raise BuildError("Extractor does not support 'stripComponents'!")
         self.dir = dir
         self.file = file
         self.strip = strip
         self.separateDownload = separateDownload
 
-    async def _extract(self, cmds, invoker):
+    async def _extract(self, cmds, invoker, stdout=None):
         destination = self.getCompressedFilePath(invoker)
         canary = destination+".extracted"
         if isYounger(destination, canary):
             for cmd in cmds:
                 if shutil.which(cmd[0]) is None: continue
-                await invoker.checkCommand(cmd, cwd=self.dir)
+                await invoker.checkCommand(cmd, cwd=self.dir, stdout=stdout)
                 invoker.trace("<touch>", canary)
                 with open(canary, "wb") as f:
                     pass
@@ -191,8 +195,7 @@ async def extract(self, invoker, destination, cwd):
 # case of tarfile broken in certain ways (e.g. tarfile will result in
 # different file modes!). But it shouldn't make a difference on Windows.
 class TarExtractor(Extractor):
-    def __init__(self, dir, file, strip, separateDownload):
-        super().__init__(dir, file, strip, separateDownload)
+    SUPPORT_STRIP = True
 
     async def extract(self, invoker):
         cmds = []
@@ -210,10 +213,6 @@ async def extract(self, invoker):
 
 
 class ZipExtractor(Extractor):
-    def __init__(self, dir, file, strip, separateDownload):
-        super().__init__(dir, file, strip, separateDownload)
-        if strip != 0:
-            raise BuildError("Extractor does not support 'stripComponents'!")
 
     async def extract(self, invoker):
         cmds = []
@@ -226,47 +225,60 @@ async def extract(self, invoker):
         await self._extract(cmds, invoker)
 
 
-class GZipExtractor(Extractor):
-    def __init__(self, dir, file, strip, separateDownload):
-        super().__init__(dir, file, strip, separateDownload)
-        if strip != 0:
-            raise BuildError("Extractor does not support 'stripComponents'!")
+class SingleFileExtractor(Extractor):
 
     async def extract(self, invoker):
-        # gunzip extracts the file at the location of the input file. Copy the
-        # downloaded file to the workspace directory prio to uncompressing it
-        cmd = ["gunzip"]
-        if self.separateDownload:
-            shutil.copyfile(self.getCompressedFilePath(invoker),
-                            invoker.joinPath(self.dir, self.file))
+        # The gunzip and unxz tools extracts the file at the location of the
+        # input file. In case the separateDownload policy is active, the
+        # destination might be in a separete folder.
+        src = self.getCompressedFilePath(invoker)
+        dst = invoker.joinPath(self.dir, self.file)
+
+        suffix = dst[-4:]
+        if self.EXT_IGNORE_CASE:
+            suffix = suffix.lower()
+
+        for ext, repl in self.EXT_MAP.items():
+            if suffix.endswith(ext):
+                dst = dst[:-len(ext)] + repl
+                break
         else:
-            cmd.append("-k")
-        cmd.extend(["-f", self.file])
-        await self._extract([cmd], invoker)
-
+            raise BuildError("unkown suffix")
+
+        with open(dst, 'wb') as f:
+            cmd = [self.CMD, "-c", src]
+            await self._extract([cmd], invoker, f)
+
+        shutil.copystat(src, dst)
+
+
+class GZipExtractor(SingleFileExtractor):
+    CMD = "gunzip"
+    EXT_IGNORE_CASE = True
+    EXT_MAP = {
+        ".gz" : "",
+        "-gz" : "",
+        ".z" : "",
+        "-z" : "",
+        "_z" : "",
+        ".tgz" : ".tar",
+        ".taz" : ".tar",
+    }
 
-class XZExtractor(Extractor):
-    def __init__(self, dir, file, strip, separateDownload):
-        super().__init__(dir, file, strip, separateDownload)
-        if strip != 0:
-            raise BuildError("Extractor does not support 'stripComponents'!")
 
-    async def extract(self, invoker):
-        cmd = ["unxz"]
-        if self.separateDownload:
-            shutil.copyfile(self.getCompressedFilePath(invoker),
-                            invoker.joinPath(self.dir, self.file))
-        else:
-            cmd.append("-k")
-        cmd.extend(["-f", self.file])
-        await self._extract([cmd], invoker)
+class XZExtractor(SingleFileExtractor):
+    CMD = "unxz"
+    EXT_IGNORE_CASE = False
+    EXT_MAP = {
+        ".xz" : "",
+        ".lzma" : "",
+        ".lz" : "",
+        ".txz" : ".tar",
+        ".tlz" : ".tar",
+    }
 
 
 class SevenZipExtractor(Extractor):
-    def __init__(self, dir, file, strip, separateDownload):
-        super().__init__(dir, file, strip, separateDownload)
-        if strip != 0:
-            raise BuildError("Extractor does not support 'stripComponents'!")
 
     async def extract(self, invoker):
         cmds = [["7z", "x", "-y", self.getCompressedFilePath(invoker)]]

test/black-box/extractors/config.yaml

@@ -1,2 +1,3 @@
+bobMinimumVersion: "0.25"
 policies:
   urlScmSeparateDownload: True

test/black-box/extractors/recipes/extract_test.yaml

@@ -40,6 +40,10 @@ buildScript: |
   for d in $(find $1/ -mindepth 1 -type d); do
     pushd $d
     sha256sum -c ${SHA256_FILE}
+    if [[ $d = */gzip || $d = */xz ]] ; then
+      # verify that extracted file retains mode
+      test $(stat -c "%a" test.dat) = "600"
+    fi
     popd
   done