Merge pull request #607 from jkloetzke/import-scm-root

Add recipe-relative import SCMs

Author: jkloetzke

doc/manual/configuration.rst

@@ -957,6 +957,7 @@ git    | ``url``: URL of remote repository
        | ``dissociate``: (Boolean, default false). Dissociate the reference (see man git-clone).
 import | ``url``: Directory path relative to project root.
        | ``prune`` (\*): Delete destination directory before importing files.
+       | ``recipeRelative`` (\*): Whether ``url`` is relative to recipe or project root. (optional)
 svn    | ``url``: URL of SVN module
        | ``revision``: Optional revision number (optional)
        | ``sslVerify`` (\*): Whether to verify the SSL certificate when fetching (optional)
@@ -1109,15 +1110,15 @@ git
 
 import
    The ``import`` SCM copies the directory specified in ``url`` to the
-   workspace. By default the destination is always overwritten and obsolete
+   workspace. By default, the destination is always overwritten and obsolete
    files are deleted. Set ``prune`` to ``False`` to only overwrite if the
    source file was changed more recently than the exiting destination in the
-   workspace. Before Bob 0.18 the default was the other way around (see
+   workspace. Before Bob 0.18, the default was the other way around (see
    :ref:`policies-pruneImportScm`).
 
-   In contrast to the other SCMs that fetch across the network the ``import``
+   In contrast to the other SCMs that fetch across the network, the ``import``
    SCM is always updated, even if ``--build-only`` is used. Because only local
-   files are imported there is no possibility to inadvertely fetch unwanted
+   files are imported, there is no possibility to inadvertently fetch unwanted
    changes from other users. The files should thus always be edited at the
    import source location and not in the workspace.
 
@@ -1126,6 +1127,11 @@ import
       content is included in the job configuration that will get too large
       otherwise.
 
+   By default, the directory given in ``url`` is interpreted relative to the
+   project root. Alternatively, ``url`` can be made relative to the recipe
+   itself if ``recipeRelative`` is set to ``True``. This is recommended
+   especially for recipes that are included as layers into other projects.
+
 svn
    The `Svn`_ SCM, like git, requires the ``url`` attribute too. If you specify a
    numeric ``revision`` Bob considers the SCM as deterministic.

pym/bob/input.py

@@ -370,7 +370,9 @@ def validate(self, data):
 
 def Scm(spec, env, overrides, recipeSet):
     # resolve with environment
-    spec = { k : ( env.substitute(v, "checkoutSCM::"+k) if isinstance(v, str) else v)
+    spec = { k : ( env.substitute(v, "checkoutSCM::"+k)
+                   if isinstance(v, str) and k not in ('__source', 'recipe')
+                   else v )
         for (k, v) in spec.items() }
 
     # apply overrides before creating scm instances. It's possible to switch the Scm type with an override..

pym/bob/scm/imp.py

@@ -116,6 +116,7 @@ class ImportScm(Scm):
     DEFAULTS = {
         schema.Optional('dir') : str,
         schema.Optional('prune') : bool,
+        schema.Optional('recipeRelative') : bool,
     }
 
     __SCHEMA = {
@@ -134,6 +135,11 @@ def __init__(self, spec, overrides=[], pruneDefault=None, fixDigestBug=False, pr
         self.__data = spec.get("__data")
         self.__projectRoot = spec.get("__projectRoot", projectRoot)
         self.__fixDigestBug = fixDigestBug
+        self.__recipeRelative = spec.get("recipeRelative", False)
+
+    def _getSrcDir(self):
+        rootDir = os.path.dirname(self._getRecipe()) if self.__recipeRelative else self.__projectRoot
+        return os.path.join(rootDir, self.__url)
 
     def getProperties(self, isJenkins, pretty=False):
         ret = super().getProperties(isJenkins)
@@ -142,9 +148,10 @@ def getProperties(self, isJenkins, pretty=False):
             'url' : self.__url,
             'dir' : self.__dir,
             'prune' : self.__prune,
+            'recipeRelative' : self.__recipeRelative,
         })
         if isJenkins:
-            ret['__data'] = packTree(self.__url)
+            ret['__data'] = packTree(self._getSrcDir())
         else:
             ret['__projectRoot'] = self.__projectRoot
         return ret
@@ -154,7 +161,7 @@ async def invoke(self, invoker):
         os.makedirs(dest, exist_ok=True)
         if self.__prune: emptyDirectory(dest)
         if self.__data is None:
-            src = os.path.join(self.__projectRoot, self.__url)
+            src = self._getSrcDir()
             if not os.path.isdir(src):
                 invoker.fail("Cannot import '{}': not a directory!".format(src))
             copyTree(src, dest, invoker)

pym/bob/scm/scm.py

@@ -231,6 +231,9 @@ def _diffSpec(self, oldScm):
         ret -= {"if"}
         return ret
 
+    def _getRecipe(self):
+        return self.__recipe
+
     def getSource(self):
         return self.__source
 

test/black-box/import-scm-relative/config.yaml

@@ -0,0 +1 @@
+bobMinimumVersion: "0.25"

test/black-box/import-scm-relative/recipes/sub/data/data.txt

@@ -0,0 +1 @@
+Hello World

test/black-box/import-scm-relative/recipes/sub/root.yaml

@@ -0,0 +1,7 @@
+root: True
+checkoutSCM:
+    scm: import
+    url: data
+    recipeRelative: True
+buildScript: cp -a "$1/"* .
+packageScript: cp -a "$1/"* .

test/black-box/import-scm-relative/run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+#
+#  Test recipeRelative "import" SCM property
+#
+. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; }
+
+cleanup
+
+# First try in-tree build
+run_bob dev sub::root
+diff -Nrq recipes/sub/data dev/dist/sub/root/1/workspace
+
+# Out of tree builds should work as well
+build="$(mktemp -d)"
+trap 'rm -rf "$build"' EXIT
+run_bob init . "$build"
+run_bob -C "$build" dev sub::root
+diff -Nrq recipes/sub/data "$build/dev/dist/sub/root/1/workspace"