Run selected checkoutScripts in build-only mode

doc/manpages/bob-build-dev.rst

@@ -181,7 +181,9 @@ Options
    defined by Bob itself (e.g. ``meta.bob``) cannot be redifined!
 
 ``-b, --build-only``
-    Don't checkout, just build and package
+    Don't checkout, just build and package. Checkout scripts whose
+    :ref:`configuration-recipes-checkoutUpdateIf` property was evaluated as
+    true will still be run.
 
     If the sources of a package that needs to be built are missing then Bob
     will still check them out. This option just prevents updates of existing

doc/manual/configuration.rst

@@ -966,7 +966,7 @@ git
    The ``rev`` property of the ``git`` SCM unifies the specification of the
    desired branch/tag/commit into one single property. If present it will be
    evaluated first. Any other ``branch``, ``tag`` or ``commit`` property is
-   evalued after it and may override a precious setting made by ``rev``. The
+   evaluated after it and may override a previous setting made by ``rev``. The
    branch/tag/commit precedence is still respected, though. Following the patterns
    described in git-rev-parse(1) the following formats are currently supported:
 
@@ -1089,6 +1089,41 @@ url
        possible to fetch multiple files in the same directory. This is done to
        separate possibly extracted files safely from other checkouts.
 
+.. _configuration-recipes-checkoutUpdateIf:
+
+checkoutUpdateIf
+~~~~~~~~~~~~~~~~
+
+Type: String | Boolean | ``null`` | IfExpression
+(:ref:`configuration-principle-booleans`), default: ``False``
+
+By default no checkout scripts are run when building with ``--build-only``.
+Some use cases practically require the ``checkoutScript`` to be always run,
+through. A typical example are code generators that generate sources from some
+high level description. These generators must be run every time when the user
+has changed the input. A recipe or class can explicitly opt in to run their
+``checkoutScript`` also in build-only mode to cover such a use case. This is
+done by either setting ``checkoutUpdateIf`` to ``True`` or by a boolean
+expression that is evaluated to ``True``. Otherwise the ``checkoutScript`` is
+ignored even if some other class enables its script. The ``checkoutUpdateIf``
+property thus only applies to the corresponding ``checkoutScript`` in the same
+recipe/class.
+
+A ``null`` value has a special semantic. It does not enable the
+``checkoutScript`` on ``--build-only`` builds by itself but only if some
+inherited class or the recipe does enable its ``checkoutUpdateIf``. This is
+useful for classes to provide some update functions but, unless an inheriting
+recipe explicitly enables ``checkoutUpdateIf``, does not cause the checkout
+step to run by itself in ``--build-only`` mode.
+
+Examples::
+
+    checkoutUpdateIf: False                             # default, same as if unset
+    checkoutUpdateIf: True                              # unconditionally run checkoutScript
+    checkoutUpdateIf: "$(is-tool-defined,idl-compiler)" # boolean expression
+    checkoutUpdateIf: !expr |                           # IfExpression
+                        is-tool-defined("idl-compiler")
+
 .. _configuration-recipes-depends:
 
 depends

pym/bob/builder.py

@@ -632,7 +632,8 @@ def linkTo(dest, linkName):
                                         "{:02}-{}".format(i, a.getPackage().getName())))
                 i += 1
 
-    async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
+    async def _runShell(self, step, scriptName, logger, cleanWorkspace=None,
+                        mode=InvocationMode.CALL):
         workspacePath = step.getWorkspacePath()
         if not os.path.isdir(workspacePath): os.makedirs(workspacePath)
         self.__linkDependencies(step)
@@ -683,7 +684,7 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
             executor=self.__executor)
         if step.jobServer() and self.__jobServer:
             invoker.setMakeParameters(self.__jobServer.getMakeFd(), self.__jobs)
-        ret = await invoker.executeStep(InvocationMode.CALL, cleanWorkspace)
+        ret = await invoker.executeStep(mode, cleanWorkspace)
         if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
         if ret == -int(signal.SIGINT):
             raise BuildError("User aborted while running {}".format(absRunFile),
@@ -695,26 +696,6 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
                                 .format(absRunFile, ret),
                              help="You may resume at this point with '--resume' after fixing the error.")
 
-    async def _runLocalSCMs(self, step, logger):
-        workspacePath = step.getWorkspacePath()
-        logFile = os.path.join(workspacePath, "..", "log.txt")
-        spec = StepSpec.fromStep(step, logFile=logFile, isJenkins=step.JENKINS)
-        invoker = Invoker(spec, self.__preserveEnv, self.__noLogFile,
-            self.__verbose >= INFO, self.__verbose >= NORMAL,
-            self.__verbose >= DEBUG, self.__bufferedStdIO,
-            executor=self.__executor)
-        ret = await invoker.executeLocalSCMs()
-        if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
-        if ret == -int(signal.SIGINT):
-            raise BuildError("User aborted while updating local SCMs",
-                             help = "Run again with '--resume' to skip already built packages.")
-        elif ret != 0:
-            if self.__bufferedStdIO:
-                logger.setError(invoker.getStdio().strip())
-            raise BuildError("Update of local SCMs failed with exit code {}"
-                                .format(ret),
-                             help="You may resume at this point with '--resume' after fixing the error.")
-
     def getStatistic(self):
         return self.__statistic
 
@@ -1029,18 +1010,23 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
             isFreshCheckout = True
         oldCheckoutHash = BobState().getResultHash(prettySrcPath)
 
+        checkoutInputHashes = [ BobState().getResultHash(i.getWorkspacePath())
+            for i in checkoutStep.getAllDepSteps() if i.isValid() ]
+
         checkoutExecuted = False
         checkoutDigest = checkoutStep.getVariantId()
         checkoutState = checkoutStep.getScmDirectories().copy()
         checkoutState[None] = (checkoutDigest, None)
         if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None):
+            inputChanged = checkoutInputHashes != BobState().getInputHashes(prettySrcPath)
+            rehash = lambda: hashWorkspace(checkoutStep)
             if not compareDirectoryState(checkoutState, oldCheckoutState):
                 stepMessage(checkoutStep, "CHECKOUT", "WARNING: recipe changed but skipped due to --build-only ({})"
                     .format(prettySrcPath), WARNING)
-            elif any((s.isLocal() and not s.isDeterministic()) for s in checkoutStep.getScmList()):
+            elif checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash):
                 with stepExec(checkoutStep, "UPDATE",
                               "{} {}".format(prettySrcPath, overridesString)) as a:
-                    await self._runLocalSCMs(checkoutStep, a)
+                    await self._runShell(checkoutStep, "checkout", a, mode=InvocationMode.UPDATE)
             else:
                 stepMessage(checkoutStep, "CHECKOUT", "skipped due to --build-only ({}) {}".format(prettySrcPath, overridesString),
                     SKIPPED, IMPORTANT)
@@ -1059,8 +1045,6 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                         # Do not use None here to distinguish it from a non-existent directory.
                         oldCheckoutState[scmDir] = (False, scmSpec)
 
-            checkoutInputHashes = [ BobState().getResultHash(i.getWorkspacePath())
-                for i in checkoutStep.getAllDepSteps() if i.isValid() ]
             if (self.__force or (not checkoutStep.isDeterministic()) or
                 (BobState().getResultHash(prettySrcPath) is None) or
                 not compareDirectoryState(checkoutState, oldCheckoutState) or

pym/bob/cmds/invoke.py

@@ -15,8 +15,8 @@ def doInvoke(argv, bobRoot):
     parser = argparse.ArgumentParser(prog="bob _invoke",
         description="Invoke a single step.")
     parser.add_argument('spec', help="The step spec file")
-    parser.add_argument('mode', default='run', choices=['run', 'shell', 'fingerprint'], nargs='?',
-        help="Invocation mode")
+    parser.add_argument('mode', default='run', choices=['run', 'update', 'shell', 'fingerprint'],
+        nargs='?', help="Invocation mode")
 
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--clean', '-c', action='store_true', default=False,
@@ -51,11 +51,12 @@ def doInvoke(argv, bobRoot):
                               False, executor=executor)
             ret = loop.run_until_complete(invoker.executeStep(InvocationMode.SHELL,
                 args.clean, args.keep_sandbox))
-        elif args.mode == 'run':
+        elif args.mode in ('run', 'update'):
             invoker = Invoker(spec, args.preserve_env, args.no_logfiles,
                 verbosity >= 2, verbosity >= 1, verbosity >= 3, False,
                 executor=executor)
-            ret = loop.run_until_complete(invoker.executeStep(InvocationMode.CALL,
+            ret = loop.run_until_complete(invoker.executeStep(
+                InvocationMode.CALL if args.mode == 'run' else InvocationMode.UPDATE,
                 args.clean, args.keep_sandbox))
         elif args.mode == 'fingerprint':
             invoker = Invoker(spec, args.preserve_env, True, True, True,

pym/bob/input.py

@@ -803,6 +803,9 @@ def getPostRunCmds(self):
     def getDigestScript(self):
         raise NotImplementedError
 
+    def getUpdateScript(self):
+        return ""
+
     def getLabel(self):
         raise NotImplementedError
 
@@ -817,6 +820,9 @@ def _getToolKeysWeak(self):
     def isDeterministic(self):
         return self.deterministic
 
+    def isUpdateDeterministic(self):
+        return True
+
     def isCheckoutStep(self):
         return False
 
@@ -1055,6 +1061,12 @@ def getDigestScript(self):
         """
         return self._coreStep.getDigestScript()
 
+    def getUpdateScript(self):
+        return self._coreStep.getUpdateScript()
+
+    def isUpdateDeterministic(self):
+        return self._coreStep.isUpdateDeterministic()
+
     def isDeterministic(self):
         """Return whether the step is deterministic.
 
@@ -1246,10 +1258,11 @@ def _getFingerprintScript(self):
 
 
 class CoreCheckoutStep(CoreStep):
-    __slots__ = ( "scmList" )
+    __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic" )
 
     def __init__(self, corePackage, checkout=None, checkoutSCMs=[],
-                 fullEnv=Env(), digestEnv=Env(), env=Env(), args=[]):
+                 fullEnv=Env(), digestEnv=Env(), env=Env(), args=[],
+                 checkoutUpdateIf=[], checkoutUpdateDeterministic=True):
         if checkout:
             recipeSet = corePackage.recipe.getRecipeSet()
             overrides = recipeSet.scmOverrides()
@@ -1273,6 +1286,8 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[],
             isValid = False
             self.scmList = []
 
+        self.__checkoutUpdateIf = checkoutUpdateIf
+        self.__checkoutUpdateDeterministic = checkoutUpdateDeterministic
         deterministic = corePackage.recipe.checkoutDeterministic
         super().__init__(corePackage, isValid, deterministic, digestEnv, env, args)
 
@@ -1294,6 +1309,9 @@ def getLabel(self):
     def isDeterministic(self):
         return super().isDeterministic() and all(s.isDeterministic() for s in self.scmList)
 
+    def isUpdateDeterministic(self):
+        return self.__checkoutUpdateDeterministic
+
     def hasLiveBuildId(self):
         return super().isDeterministic() and all(s.hasLiveBuildId() for s in self.scmList)
 
@@ -1324,6 +1342,10 @@ def getDigestScript(self):
         else:
             return None
 
+    def getUpdateScript(self):
+        glue = getLanguage(self.corePackage.recipe.scriptLanguage.index).glue
+        return joinScripts(self.__checkoutUpdateIf, glue) or ""
+
     @property
     def fingerprintMask(self):
         return 0
@@ -1493,8 +1515,10 @@ def refDeref(self, stack, inputTools, inputSandbox, pathFormatter):
         tools, sandbox = self.internalRef.refDeref(stack, inputTools, inputSandbox, pathFormatter)
         return Package(self, stack, pathFormatter, inputTools, tools, inputSandbox, sandbox)
 
-    def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args):
-        ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args)
+    def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv,
+                               env, args, checkoutUpdateIf, checkoutUpdateDeterministic):
+        ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs,
+            fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic)
         return ret
 
     def createInvalidCoreCheckoutStep(self):
@@ -1933,6 +1957,7 @@ def createVirtualRoot(recipeSet, roots, properties):
             "depends" : [
                 { "name" : name, "use" : ["result"] } for name in roots
             ],
+            "checkoutUpdateIf" : False,
             "buildScript" : "true",
             "packageScript" : "true"
         }
@@ -2015,6 +2040,7 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         for a in self.__checkoutAsserts:
             a["__source"] = sourceName + ", checkoutAssert #{}".format(i)
             i += 1
+        self.__checkoutUpdateIf = recipe["checkoutUpdateIf"]
         self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh)
         self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh)
         self.__fingerprintScriptList = fetchFingerprintScripts(recipe)
@@ -2089,15 +2115,25 @@ def coDet(r):
             if ret is not None:
                 return ret
             return r.__checkout[self.__scriptLanguage][1][0] is None
-        self.__checkoutDeterministic = all(coDet(i) for i in inheritAll)
+
+        checkoutDeterministic = [ coDet(i) for i in inheritAll ]
+        self.__checkoutDeterministic = all(checkoutDeterministic)
 
         # merge scripts and other lists
         selLang = lambda x: x[self.__scriptLanguage]
 
         # Join all scripts. The result is a tuple with (setupScript, mainScript, digestScript)
-        self.__checkout = mergeScripts([ selLang(i.__checkout) for i in inheritAll ], glue)
+        checkoutScripts = [ selLang(i.__checkout) for i in inheritAll ]
+        self.__checkout = mergeScripts(checkoutScripts, glue)
         self.__checkoutSCMs = list(chain.from_iterable(i.__checkoutSCMs for i in inheritAll))
         self.__checkoutAsserts = list(chain.from_iterable(i.__checkoutAsserts for i in inheritAll))
+        self.__checkoutUpdateIf = [
+            (cond, fragments[1][0], deterministic)
+                for cond, fragments, deterministic
+                in zip((i.__checkoutUpdateIf for i in inheritAll), checkoutScripts,
+                       checkoutDeterministic)
+                if cond != False
+        ]
         self.__build = mergeScripts([ selLang(i.__build) for i in inheritAll ], glue)
         self.__package = mergeScripts([ selLang(i.__package) for i in inheritAll ], glue)
         self.__fingerprintScriptList = [ i.__fingerprintScriptList for i in inheritAll ]
@@ -2511,9 +2547,24 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None,
             checkoutDigestEnv = env.prune(self.__checkoutVars)
             checkoutEnv = ( env.prune(self.__checkoutVars | self.__checkoutVarsWeak)
                 if self.__checkoutVarsWeak else checkoutDigestEnv )
+            checkoutUpdateIf = [
+                ( (env.evaluate(cond, "checkoutUpdateIf")
+                    if (isinstance(cond, str) or isinstance(cond, IfExpression))
+                    else cond),
+                  script,
+                  deterministic)
+                for cond, script, deterministic in self.__checkoutUpdateIf ]
+            if any(cond == True for cond, _, _ in checkoutUpdateIf):
+                checkoutUpdateDeterministic = all(
+                    deterministic for cond, _, deterministic in checkoutUpdateIf
+                    if cond != False)
+                checkoutUpdateIf = [ script for cond, script, _ in checkoutUpdateIf if cond != False ]
+            else:
+                checkoutUpdateDeterministic = True
+                checkoutUpdateIf = []
             srcCoreStep = p.createCoreCheckoutStep(self.__checkout,
                 self.__checkoutSCMs, env, checkoutDigestEnv, checkoutEnv,
-                checkoutDeps)
+                checkoutDeps, checkoutUpdateIf, checkoutUpdateDeterministic)
         else:
             srcCoreStep = p.createInvalidCoreCheckoutStep()
 
@@ -3560,6 +3611,7 @@ def __createSchemas(self):
             schema.Optional('checkoutSetup') : str,
             schema.Optional('checkoutSetupBash') : str,
             schema.Optional('checkoutSetupPwsh') : str,
+            schema.Optional('checkoutUpdateIf', default=False) : schema.Or(None, str, bool, IfExpression),
             schema.Optional('buildScript') : str,
             schema.Optional('buildScriptBash') : str,
             schema.Optional('buildScriptPwsh') : str,

pym/bob/intermediate.py

@@ -85,10 +85,12 @@ def fromStep(cls, step, graph, partial=False):
             self.__data['postRunCmds'] = step.getPostRunCmds()
             self.__data['setupScript'] = step.getSetupScript()
             self.__data['mainScript'] = step.getMainScript()
+            self.__data['updateScript'] = step.getUpdateScript()
             self.__data['fingerprintScript'] = step._getFingerprintScript()
             self.__data['jobServer'] = step.jobServer()
             self.__data['label'] = step.getLabel()
             self.__data['isDeterministic'] = step.isDeterministic()
+            self.__data['isUpdateDeterministic'] = step.isUpdateDeterministic()
             self.__data['hasNetAccess'] = step.hasNetAccess()
             if self.__data['isCheckoutStep']:
                 self.__data['hasLiveBuildId'] = step.hasLiveBuildId()
@@ -255,6 +257,9 @@ def getSetupScript(self):
     def getMainScript(self):
         return self.__data['mainScript']
 
+    def getUpdateScript(self):
+        return self.__data['updateScript']
+
     def _getFingerprintScript(self):
         return self.__data['fingerprintScript']
 
@@ -267,6 +272,9 @@ def getLabel(self):
     def isDeterministic(self):
         return self.__data['isDeterministic']
 
+    def isUpdateDeterministic(self):
+        return self.__data['isUpdateDeterministic']
+
     def hasLiveBuildId(self):
         return self.__data['hasLiveBuildId']
 
@@ -285,6 +293,15 @@ def deserialize(state):
     def getScmDirectories(self):
         return { d : (bytes.fromhex(h), p) for (d, (h, p)) in self.__data['scmDirectories'].items() }
 
+    def mayUpdate(self, inputChanged, oldHash, rehash):
+        if any((s.isLocal() and not s.isDeterministic()) for s in self.getScmList()):
+            return True
+        if not self.getUpdateScript():
+            return False
+        if not self.isUpdateDeterministic():
+            return True
+        return rehash() != oldHash
+
     def _getSandboxVariantId(self):
         return bytes.fromhex(self.__data['sandboxVariantId'])