Merge pull request #596 from jkloetzke/layer-compat

Restore pre-0.25 nested layers compatibility

Author: jkloetzke

doc/manual/configuration.rst

@@ -1968,6 +1968,10 @@ possibility is to provide an SCM-Dictionary (see
           commit: ...
         - bsp
 
+.. note::
+   Managed layers are only supported if the :ref:`policies-managedLayers`
+   policy is set to the new behaviour.
+
 If a layer SCM specification is given, Bob takes care of the layer management:
 
 - Layers are checked out / updated during bob-build (except build-only).

doc/manual/policies.rst

@@ -290,6 +290,60 @@ Old behavior
 New behavior
     Apply string substitution to ``metaEnvironment`` variables.
 
+.. _policies-managedLayers:
+
+managedLayers
+~~~~~~~~~~~~~
+
+Introduced in: 0.25
+
+Starting with Bob version 0.25, managed layers are supported. This changed the
+location where layers are stored, though. Historically, layers could be nested
+where they would form a tree structure. That is, each layer can have a ``layers``
+directory itself where further layers are located. Because this does not work
+if multiple layers refer to another common layer, the directory structure
+has been flattened.
+
+Old behavior
+    Keep support for projects that were created before Bob 0.25. Layers with
+    sub-layers form a tree structure. See the following example::
+
+        .
+        ├── config.yaml
+        ├── layers
+        │   └── foo
+        │       ├── config.yaml
+        │       ├── layers
+        │       │   ├── bar
+        │       │   │   └── recipes
+        │       │   └── baz
+        │       │       └── recipes
+        │       └── recipes
+        └── recipes
+
+    No SCM can be used in the :ref:`configuration-config-layers` section of
+    ``config.yaml``. The :ref:`manpage-layers` command will refuse to work on
+    such projectes.
+
+New behavior
+    Managed layers are supported, that is SCMs can be used in the
+    :ref:`configuration-config-layers` section of ``config.yaml``. The layers
+    are checked out flat into the ``layers`` directory of the project::
+
+        .
+        ├── config.yaml
+        ├── layers
+        │   ├── bar
+        │   │   └── recipes
+        │   ├── baz
+        │   │   └── recipes
+        │   └── foo
+        │       ├── config.yaml
+        │       └── recipes
+        └── recipes
+
+    Unmanaged layers are expected in the same directory.
+
 .. _policies-obsolete:
 
 Obsolete policies

pym/bob/builder.py

@@ -1479,7 +1479,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId):
           * still same build-id -> done
           * build-id changed -> prune and try download, fall back to build
         """
-        layer = packageStep.getPackage().getRecipe().getLayer()
+        layer = "/".join(packageStep.getPackage().getRecipe().getLayer())
         layerDownloadMode = None
         if layer:
             for mode in self.__downloadLayerModes:

pym/bob/cmds/build/build.py

@@ -238,7 +238,7 @@ def _downloadLayerArgument(arg):
         recipes.setConfigFiles(args.configFile)
         if args.build_mode != 'build-only':
             setVerbosity(args.verbose)
-            updateLayers(loop, defines, args.verbose, args.attic, args.layerConfig)
+            updateLayers(loop, defines, args.verbose, args.attic, args.layerConfig, False)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue

pym/bob/input.py

@@ -2087,9 +2087,16 @@ def getLayer(self):
         are named from top to bottom. Example:
         ``layers/foo/layers/bar/recipes/baz.yaml`` -> ``['foo', 'bar']``.
 
+        If the managedLayers policy is set to the new behaviour, nested layers
+        are flattened. This means that layers are always returnd as single-item
+        lists.
+
         :rtype: List[str]
         """
-        return self.__layer
+        if self.__layer:
+            return self.__layer.split("/")
+        else:
+            return []
 
     def resolveClasses(self, rootEnv):
         # must be done only once
@@ -3003,6 +3010,7 @@ class RecipeSet:
                 schema.Optional('fixImportScmVariant') : bool,
                 schema.Optional('defaultFileMode') : bool,
                 schema.Optional('substituteMetaEnv') : bool,
+                schema.Optional('managedLayers') : bool,
             },
             error="Invalid policy specified! Are you using an appropriate version of Bob?"
         ),
@@ -3051,7 +3059,12 @@ class RecipeSet:
             "0.25rc1",
             InfoOnce("substituteMetaEnv policy is not set. MetaEnv will not be substituted.",
                 help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#substitutemetaenv for more information.")
-        )
+        ),
+        "managedLayers": (
+            "0.25.0rc2.dev6",
+            InfoOnce("managedLayers policy is not set. Only unmanaged layers are supported.",
+                help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#managedlayers for more information.")
+        ),
     }
 
     _ignoreCmdConfig = False
@@ -3481,7 +3494,7 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
                 os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml'))
 
         # Begin with root layer
-        self.__parseLayer(LayerSpec(""), "9999", recipesRoot)
+        self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None)
 
         # Out-of-tree builds may have a dedicated default.yaml
         if recipesRoot:
@@ -3557,20 +3570,40 @@ def calculatePolicies(cls, config):
             ret[name] = (behaviour, None)
         return ret
 
-    def __parseLayer(self, layerSpec, maxVer, recipesRoot):
+    def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer):
         layer = layerSpec.getName()
-
-        if layer in self.__layers:
-            return
-        self.__layers.append(layer)
-
-        # SCM backed layers are in build dir, regular layers are in project dir.
-        rootDir = recipesRoot if layerSpec.getScm() is None else ""
         if layer:
-            rootDir = os.path.join(rootDir, "layers", layer)
-        if not os.path.isdir(rootDir or "."):
-            raise ParseError(f"Layer '{layer}' does not exist!",
+            # Managed layers imply that layers are potentially nested instead
+            # of being checked out next to each other in the build directory.
+            managedLayers = self.getPolicy('managedLayers')
+            if layerSpec.getScm() is not None and not managedLayers:
+                raise ParseError("Managed layers aren't enabled! See the managedLayers policy for details.")
+
+            # Pre 0.25, layers could be nested.
+            if not managedLayers and upperLayer:
+                layer = upperLayer + "/" + layer
+
+            if layer in self.__layers:
+                return
+            self.__layers.append(layer)
+
+            if managedLayers:
+                # SCM backed layers are in build dir, regular layers are in
+                # project dir.
+                rootDir = recipesRoot if layerSpec.getScm() is None else ""
+                rootDir = os.path.join(rootDir, "layers", layer)
+                if not os.path.isdir(rootDir):
+                    raise ParseError(f"Layer '{layer}' does not exist!",
                                      help="You probably want to run 'bob layers update' to fetch missing layers.")
+            else:
+                # Before managed layers existed, layers could be nested in the
+                # project directory.
+                rootDir = os.path.join(recipesRoot, *( os.path.join("layers", l)
+                                                       for l in layer.split("/") ))
+                if not os.path.isdir(rootDir):
+                    raise ParseError(f"Layer '{layer}' does not exist!")
+        else:
+            rootDir = recipesRoot
 
         config = self.loadConfigYaml(self.loadYaml, rootDir)
         minVer = config.get("bobMinimumVersion", "0.16")
@@ -3593,7 +3626,7 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot):
         # First parse any sub-layers. Their settings have a lower precedence
         # and may be overwritten by higher layers.
         for l in config.get("layers", []):
-            self.__parseLayer(l, maxVer, recipesRoot)
+            self.__parseLayer(l, maxVer, recipesRoot, layer)
 
         # Load plugins and re-create schemas as new keys may have been added
         self.__loadPlugins(rootDir, layer, config.get("plugins", []))

pym/bob/layers.py

@@ -3,7 +3,7 @@
 import schema
 import shutil
 from textwrap import indent
-from .errors import BuildError
+from .errors import BuildError, ParseError
 from .invoker import CmdFailedError, InvocationError, Invoker
 from .scm import getScm, ScmOverride, ScmStatus, ScmTaint
 from .state import BobState
@@ -205,6 +205,9 @@ def getSubLayers(self):
     def getScm(self):
         return self.__scm
 
+    def getPolicy(self, name, location=None):
+        return self.__config.getPolicy(name, location)
+
 class Layers:
     def __init__(self, defines, attic):
         self.__layers = {}
@@ -260,7 +263,7 @@ def cleanupUnused(self):
                 os.rename(d, atticPath)
                 BobState().delLayerState(d)
 
-    def collect(self, loop, update, verbose=0):
+    def collect(self, loop, update, verbose=0, requireManagedLayers=True):
         configSchema = (schema.Schema(RecipeSet.STATIC_CONFIG_LAYER_SPEC), b'')
         config = LayersConfig()
         with YamlCache() as yamlCache:
@@ -273,9 +276,16 @@ def collect(self, loop, update, verbose=0):
 
             rootLayers = Layer("", config, self.__defines, self.__projectRoot)
             rootLayers.parse(yamlCache)
+            if not rootLayers.getPolicy("managedLayers"):
+                if requireManagedLayers:
+                    raise ParseError("Managed layers aren't enabled! See the managedLayers policy for details.")
+                else:
+                    return False
             self.__layers[0] = rootLayers.getSubLayers();
             self.__collect(loop, 0, yamlCache, update, verbose)
 
+        return True
+
     def setLayerConfig(self, configFiles):
         self.__layerConfigFiles = configFiles
 
@@ -326,8 +336,8 @@ def status(self, printer):
         for (layerDir, status) in sorted(result.items()):
             printer(status, layerDir)
 
-def updateLayers(loop, defines, verbose, attic, layerConfigs):
+def updateLayers(loop, defines, verbose, attic, layerConfigs, requireManagedLayers=True):
     layers = Layers(defines, attic)
     layers.setLayerConfig(layerConfigs)
-    layers.collect(loop, True, verbose)
-    layers.cleanupUnused()
+    if layers.collect(loop, True, verbose, requireManagedLayers):
+        layers.cleanupUnused()

test/black-box/layers-checkout/config.yaml

@@ -1,3 +1,4 @@
+bobMinimumVersion: "0.25.0rc2.dev6"
 layers:
   - name: foo
     scm: git

test/black-box/layers-checkout/run.sh

@@ -188,6 +188,12 @@ run_bob layers update -DBAR_1_COMMIT=${bar_c1} -DBAR_2_COMMIT=${bar_c1} -DBAR_DI
 	-lc layers_overrides -vv
 expect_exist layers/foo/override
 
+# test that layers status/update are rejected on the old managedLayers policy
+old_dir="$tmp_dir/legacy"
+mkdir -p "$old_dir/recipes"
+expect_fail run_bob -C "$old_dir" layers update
+expect_fail run_bob -C "$old_dir" layers status
+
 # remove layers + clean
 cleanup
 rm -rf layers

test/unit/test_input_recipeset.py

@@ -34,25 +34,24 @@ def tearDown(self):
         os.chdir(self.cwd)
         self.tmpdir.cleanup()
 
-    def writeRecipe(self, name, content, layer=None):
+    def writeRecipe(self, name, content, layer=[]):
         path = os.path.join("",
-            os.path.join("layers", layer) if layer is not None else "",
+            *(os.path.join("layers", l) for l in layer),
             "recipes")
         if path: os.makedirs(path, exist_ok=True)
         with open(os.path.join(path, name+".yaml"), "w") as f:
             f.write(textwrap.dedent(content))
 
-    def writeClass(self, name, content, layer=None):
+    def writeClass(self, name, content, layer=[]):
         path = os.path.join("",
-            os.path.join("layers", layer) if layer is not None else "",
+            *(os.path.join("layers", l) for l in layer),
             "classes")
         if path: os.makedirs(path, exist_ok=True)
         with open(os.path.join(path, name+".yaml"), "w") as f:
             f.write(textwrap.dedent(content))
 
-    def writeConfig(self, content, layer=None):
-        path = os.path.join("",
-            os.path.join("layers", layer) if layer is not None else "")
+    def writeConfig(self, content, layer=[]):
+        path = os.path.join("", *(os.path.join("layers", l) for l in layer))
         if path: os.makedirs(path, exist_ok=True)
         with open(os.path.join(path, "config.yaml"), "w") as f:
             f.write(yaml.dump(content))
@@ -1343,26 +1342,26 @@ def setUp(self):
         self.writeConfig({
             "bobMinimumVersion" : "0.24",
             "layers" : [ "l2" ],
-        }, layer="l1_n1")
+        }, layer=["l1_n1"])
         self.writeRecipe("foo", """\
             depends:
                 - baz
             buildScript: "true"
             packageScript: "true"
             """,
-            layer="l1_n1")
+            layer=["l1_n1"])
 
         self.writeRecipe("baz", """\
             buildScript: "true"
             packageScript: "true"
             """,
-            layer="l2")
+            layer=["l1_n1", "l2"])
 
         self.writeRecipe("bar", """\
             buildScript: "true"
             packageScript: "true"
             """,
-            layer="l1_n2")
+            layer=["l1_n2"])
 
     def testRegular(self):
         """Test that layers can be parsed"""
@@ -1376,13 +1375,13 @@ def testRecipeObstruction(self):
             buildScript: "true"
             packageScript: "true"
             """,
-            layer="l1_n2")
+            layer=["l1_n2"])
         self.assertRaises(ParseError, self.generate)
 
     def testClassObstruction(self):
         """Test that layers must not provide identical classes"""
-        self.writeClass("c", "", layer="l2")
-        self.writeClass("c", "", layer="l1_n2")
+        self.writeClass("c", "", layer=["l1_n1", "l2"])
+        self.writeClass("c", "", layer=["l1_n2"])
         self.assertRaises(ParseError, self.generate)
 
     def testMinimumVersion(self):
@@ -1393,6 +1392,22 @@ def testMinimumVersion(self):
         })
         self.assertRaises(ParseError, self.generate)
 
+    def testManagedRejected(self):
+        self.writeConfig({
+            "bobMinimumVersion" : "0.25rc1",
+            "layers" : [
+                {
+                    "name" : "l1_n1",
+                    "scm" : "git",
+                    "url" : "[email protected]:bob.git",
+                }
+            ]
+        })
+        with self.assertRaises(ParseError) as err:
+            self.generate()
+        self.assertEqual(err.exception.slogan,
+                         "Managed layers aren't enabled! See the managedLayers policy for details.")
+
 class TestIfExpression(RecipesTmp, TestCase):
     """ Test if expressions """
     def setUp(self):