build: add checkout update facility in build-only mode

By default no checkout scripts are run when building with --build-only.
Some use cases practically require the checkoutScript to be always run,
through. A typical example are code generators that generate sources
from some high level description. These generators must be run every
time when the user has changed the input. A recipe or class can
explicitly opt in to run their checkoutScript also in build-only mode to
cover such a use case. This is done by either setting checkoutUpdateIf
to True or by a boolean expression that is evaluated to True. Otherwise
the checkoutScript is ignored even if some other class enables its
script. The checkoutUpdateIf property thus only applies to the
corresponding checkoutScript in the same recipe/class.

The build logic already did support the import SCM which was run in
build-only mode too. Unify the handling of the import SCM with the
selective enabling of checkoutScript's on build-only builds.

Fixes #452.

pym/bob/builder.py

@@ -632,7 +632,8 @@ def linkTo(dest, linkName):
                                         "{:02}-{}".format(i, a.getPackage().getName())))
                 i += 1
 
-    async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
+    async def _runShell(self, step, scriptName, logger, cleanWorkspace=None,
+                        mode=InvocationMode.CALL):
         workspacePath = step.getWorkspacePath()
         if not os.path.isdir(workspacePath): os.makedirs(workspacePath)
         self.__linkDependencies(step)
@@ -683,7 +684,7 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
             executor=self.__executor)
         if step.jobServer() and self.__jobServer:
             invoker.setMakeParameters(self.__jobServer.getMakeFd(), self.__jobs)
-        ret = await invoker.executeStep(InvocationMode.CALL, cleanWorkspace)
+        ret = await invoker.executeStep(mode, cleanWorkspace)
         if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
         if ret == -int(signal.SIGINT):
             raise BuildError("User aborted while running {}".format(absRunFile),
@@ -695,26 +696,6 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None):
                                 .format(absRunFile, ret),
                              help="You may resume at this point with '--resume' after fixing the error.")
 
-    async def _runLocalSCMs(self, step, logger):
-        workspacePath = step.getWorkspacePath()
-        logFile = os.path.join(workspacePath, "..", "log.txt")
-        spec = StepSpec.fromStep(step, logFile=logFile, isJenkins=step.JENKINS)
-        invoker = Invoker(spec, self.__preserveEnv, self.__noLogFile,
-            self.__verbose >= INFO, self.__verbose >= NORMAL,
-            self.__verbose >= DEBUG, self.__bufferedStdIO,
-            executor=self.__executor)
-        ret = await invoker.executeLocalSCMs()
-        if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
-        if ret == -int(signal.SIGINT):
-            raise BuildError("User aborted while updating local SCMs",
-                             help = "Run again with '--resume' to skip already built packages.")
-        elif ret != 0:
-            if self.__bufferedStdIO:
-                logger.setError(invoker.getStdio().strip())
-            raise BuildError("Update of local SCMs failed with exit code {}"
-                                .format(ret),
-                             help="You may resume at this point with '--resume' after fixing the error.")
-
     def getStatistic(self):
         return self.__statistic
 
@@ -1029,18 +1010,23 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
             isFreshCheckout = True
         oldCheckoutHash = BobState().getResultHash(prettySrcPath)
 
+        checkoutInputHashes = [ BobState().getResultHash(i.getWorkspacePath())
+            for i in checkoutStep.getAllDepSteps() if i.isValid() ]
+
         checkoutExecuted = False
         checkoutDigest = checkoutStep.getVariantId()
         checkoutState = checkoutStep.getScmDirectories().copy()
         checkoutState[None] = (checkoutDigest, None)
         if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None):
+            inputChanged = checkoutInputHashes != BobState().getInputHashes(prettySrcPath)
+            rehash = lambda: hashWorkspace(checkoutStep)
             if not compareDirectoryState(checkoutState, oldCheckoutState):
                 stepMessage(checkoutStep, "CHECKOUT", "WARNING: recipe changed but skipped due to --build-only ({})"
                     .format(prettySrcPath), WARNING)
-            elif any((s.isLocal() and not s.isDeterministic()) for s in checkoutStep.getScmList()):
+            elif checkoutStep.mayUpdate(inputChanged, BobState().getResultHash(prettySrcPath), rehash):
                 with stepExec(checkoutStep, "UPDATE",
                               "{} {}".format(prettySrcPath, overridesString)) as a:
-                    await self._runLocalSCMs(checkoutStep, a)
+                    await self._runShell(checkoutStep, "checkout", a, mode=InvocationMode.UPDATE)
             else:
                 stepMessage(checkoutStep, "CHECKOUT", "skipped due to --build-only ({}) {}".format(prettySrcPath, overridesString),
                     SKIPPED, IMPORTANT)
@@ -1059,8 +1045,6 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                         # Do not use None here to distinguish it from a non-existent directory.
                         oldCheckoutState[scmDir] = (False, scmSpec)
 
-            checkoutInputHashes = [ BobState().getResultHash(i.getWorkspacePath())
-                for i in checkoutStep.getAllDepSteps() if i.isValid() ]
             if (self.__force or (not checkoutStep.isDeterministic()) or
                 (BobState().getResultHash(prettySrcPath) is None) or
                 not compareDirectoryState(checkoutState, oldCheckoutState) or

pym/bob/cmds/invoke.py

@@ -15,8 +15,8 @@ def doInvoke(argv, bobRoot):
     parser = argparse.ArgumentParser(prog="bob _invoke",
         description="Invoke a single step.")
     parser.add_argument('spec', help="The step spec file")
-    parser.add_argument('mode', default='run', choices=['run', 'shell', 'fingerprint'], nargs='?',
-        help="Invocation mode")
+    parser.add_argument('mode', default='run', choices=['run', 'update', 'shell', 'fingerprint'],
+        nargs='?', help="Invocation mode")
 
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--clean', '-c', action='store_true', default=False,
@@ -51,11 +51,12 @@ def doInvoke(argv, bobRoot):
                               False, executor=executor)
             ret = loop.run_until_complete(invoker.executeStep(InvocationMode.SHELL,
                 args.clean, args.keep_sandbox))
-        elif args.mode == 'run':
+        elif args.mode in ('run', 'update'):
             invoker = Invoker(spec, args.preserve_env, args.no_logfiles,
                 verbosity >= 2, verbosity >= 1, verbosity >= 3, False,
                 executor=executor)
-            ret = loop.run_until_complete(invoker.executeStep(InvocationMode.CALL,
+            ret = loop.run_until_complete(invoker.executeStep(
+                InvocationMode.CALL if args.mode == 'run' else InvocationMode.UPDATE,
                 args.clean, args.keep_sandbox))
         elif args.mode == 'fingerprint':
             invoker = Invoker(spec, args.preserve_env, True, True, True,

pym/bob/input.py

@@ -803,6 +803,9 @@ def getPostRunCmds(self):
     def getDigestScript(self):
         raise NotImplementedError
 
+    def getUpdateScript(self):
+        return ""
+
     def getLabel(self):
         raise NotImplementedError
 
@@ -817,6 +820,9 @@ def _getToolKeysWeak(self):
     def isDeterministic(self):
         return self.deterministic
 
+    def isUpdateDeterministic(self):
+        return True
+
     def isCheckoutStep(self):
         return False
 
@@ -1055,6 +1061,12 @@ def getDigestScript(self):
         """
         return self._coreStep.getDigestScript()
 
+    def getUpdateScript(self):
+        return self._coreStep.getUpdateScript()
+
+    def isUpdateDeterministic(self):
+        return self._coreStep.isUpdateDeterministic()
+
     def isDeterministic(self):
         """Return whether the step is deterministic.
 
@@ -1246,10 +1258,11 @@ def _getFingerprintScript(self):
 
 
 class CoreCheckoutStep(CoreStep):
-    __slots__ = ( "scmList" )
+    __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic" )
 
     def __init__(self, corePackage, checkout=None, checkoutSCMs=[],
-                 fullEnv=Env(), digestEnv=Env(), env=Env(), args=[]):
+                 fullEnv=Env(), digestEnv=Env(), env=Env(), args=[],
+                 checkoutUpdateIf=[], checkoutUpdateDeterministic=True):
         if checkout:
             recipeSet = corePackage.recipe.getRecipeSet()
             overrides = recipeSet.scmOverrides()
@@ -1273,6 +1286,8 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[],
             isValid = False
             self.scmList = []
 
+        self.__checkoutUpdateIf = checkoutUpdateIf
+        self.__checkoutUpdateDeterministic = checkoutUpdateDeterministic
         deterministic = corePackage.recipe.checkoutDeterministic
         super().__init__(corePackage, isValid, deterministic, digestEnv, env, args)
 
@@ -1294,6 +1309,9 @@ def getLabel(self):
     def isDeterministic(self):
         return super().isDeterministic() and all(s.isDeterministic() for s in self.scmList)
 
+    def isUpdateDeterministic(self):
+        return self.__checkoutUpdateDeterministic
+
     def hasLiveBuildId(self):
         return super().isDeterministic() and all(s.hasLiveBuildId() for s in self.scmList)
 
@@ -1324,6 +1342,10 @@ def getDigestScript(self):
         else:
             return None
 
+    def getUpdateScript(self):
+        glue = getLanguage(self.corePackage.recipe.scriptLanguage.index).glue
+        return joinScripts(self.__checkoutUpdateIf, glue) or ""
+
     @property
     def fingerprintMask(self):
         return 0
@@ -1493,8 +1515,10 @@ def refDeref(self, stack, inputTools, inputSandbox, pathFormatter):
         tools, sandbox = self.internalRef.refDeref(stack, inputTools, inputSandbox, pathFormatter)
         return Package(self, stack, pathFormatter, inputTools, tools, inputSandbox, sandbox)
 
-    def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args):
-        ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args)
+    def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv,
+                               env, args, checkoutUpdateIf, checkoutUpdateDeterministic):
+        ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs,
+            fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic)
         return ret
 
     def createInvalidCoreCheckoutStep(self):
@@ -1933,6 +1957,7 @@ def createVirtualRoot(recipeSet, roots, properties):
             "depends" : [
                 { "name" : name, "use" : ["result"] } for name in roots
             ],
+            "checkoutUpdateIf" : False,
             "buildScript" : "true",
             "packageScript" : "true"
         }
@@ -2015,6 +2040,7 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         for a in self.__checkoutAsserts:
             a["__source"] = sourceName + ", checkoutAssert #{}".format(i)
             i += 1
+        self.__checkoutUpdateIf = recipe["checkoutUpdateIf"]
         self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh)
         self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh)
         self.__fingerprintScriptList = fetchFingerprintScripts(recipe)
@@ -2089,15 +2115,25 @@ def coDet(r):
             if ret is not None:
                 return ret
             return r.__checkout[self.__scriptLanguage][1][0] is None
-        self.__checkoutDeterministic = all(coDet(i) for i in inheritAll)
+
+        checkoutDeterministic = [ coDet(i) for i in inheritAll ]
+        self.__checkoutDeterministic = all(checkoutDeterministic)
 
         # merge scripts and other lists
         selLang = lambda x: x[self.__scriptLanguage]
 
         # Join all scripts. The result is a tuple with (setupScript, mainScript, digestScript)
-        self.__checkout = mergeScripts([ selLang(i.__checkout) for i in inheritAll ], glue)
+        checkoutScripts = [ selLang(i.__checkout) for i in inheritAll ]
+        self.__checkout = mergeScripts(checkoutScripts, glue)
         self.__checkoutSCMs = list(chain.from_iterable(i.__checkoutSCMs for i in inheritAll))
         self.__checkoutAsserts = list(chain.from_iterable(i.__checkoutAsserts for i in inheritAll))
+        self.__checkoutUpdateIf = [
+            (cond, fragments[1][0], deterministic)
+                for cond, fragments, deterministic
+                in zip((i.__checkoutUpdateIf for i in inheritAll), checkoutScripts,
+                       checkoutDeterministic)
+                if cond != False
+        ]
         self.__build = mergeScripts([ selLang(i.__build) for i in inheritAll ], glue)
         self.__package = mergeScripts([ selLang(i.__package) for i in inheritAll ], glue)
         self.__fingerprintScriptList = [ i.__fingerprintScriptList for i in inheritAll ]
@@ -2511,9 +2547,24 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None,
             checkoutDigestEnv = env.prune(self.__checkoutVars)
             checkoutEnv = ( env.prune(self.__checkoutVars | self.__checkoutVarsWeak)
                 if self.__checkoutVarsWeak else checkoutDigestEnv )
+            checkoutUpdateIf = [
+                ( (env.evaluate(cond, "checkoutUpdateIf")
+                    if (isinstance(cond, str) or isinstance(cond, IfExpression))
+                    else cond),
+                  script,
+                  deterministic)
+                for cond, script, deterministic in self.__checkoutUpdateIf ]
+            if any(cond == True for cond, _, _ in checkoutUpdateIf):
+                checkoutUpdateDeterministic = all(
+                    deterministic for cond, _, deterministic in checkoutUpdateIf
+                    if cond != False)
+                checkoutUpdateIf = [ script for cond, script, _ in checkoutUpdateIf if cond != False ]
+            else:
+                checkoutUpdateDeterministic = True
+                checkoutUpdateIf = []
             srcCoreStep = p.createCoreCheckoutStep(self.__checkout,
                 self.__checkoutSCMs, env, checkoutDigestEnv, checkoutEnv,
-                checkoutDeps)
+                checkoutDeps, checkoutUpdateIf, checkoutUpdateDeterministic)
         else:
             srcCoreStep = p.createInvalidCoreCheckoutStep()
 
@@ -3560,6 +3611,7 @@ def __createSchemas(self):
             schema.Optional('checkoutSetup') : str,
             schema.Optional('checkoutSetupBash') : str,
             schema.Optional('checkoutSetupPwsh') : str,
+            schema.Optional('checkoutUpdateIf', default=False) : schema.Or(None, str, bool, IfExpression),
             schema.Optional('buildScript') : str,
             schema.Optional('buildScriptBash') : str,
             schema.Optional('buildScriptPwsh') : str,

pym/bob/intermediate.py

@@ -85,10 +85,12 @@ def fromStep(cls, step, graph, partial=False):
             self.__data['postRunCmds'] = step.getPostRunCmds()
             self.__data['setupScript'] = step.getSetupScript()
             self.__data['mainScript'] = step.getMainScript()
+            self.__data['updateScript'] = step.getUpdateScript()
             self.__data['fingerprintScript'] = step._getFingerprintScript()
             self.__data['jobServer'] = step.jobServer()
             self.__data['label'] = step.getLabel()
             self.__data['isDeterministic'] = step.isDeterministic()
+            self.__data['isUpdateDeterministic'] = step.isUpdateDeterministic()
             self.__data['hasNetAccess'] = step.hasNetAccess()
             if self.__data['isCheckoutStep']:
                 self.__data['hasLiveBuildId'] = step.hasLiveBuildId()
@@ -255,6 +257,9 @@ def getSetupScript(self):
     def getMainScript(self):
         return self.__data['mainScript']
 
+    def getUpdateScript(self):
+        return self.__data['updateScript']
+
     def _getFingerprintScript(self):
         return self.__data['fingerprintScript']
 
@@ -267,6 +272,9 @@ def getLabel(self):
     def isDeterministic(self):
         return self.__data['isDeterministic']
 
+    def isUpdateDeterministic(self):
+        return self.__data['isUpdateDeterministic']
+
     def hasLiveBuildId(self):
         return self.__data['hasLiveBuildId']
 
@@ -285,6 +293,15 @@ def deserialize(state):
     def getScmDirectories(self):
         return { d : (bytes.fromhex(h), p) for (d, (h, p)) in self.__data['scmDirectories'].items() }
 
+    def mayUpdate(self, inputChanged, oldHash, rehash):
+        if any((s.isLocal() and not s.isDeterministic()) for s in self.getScmList()):
+            return True
+        if not self.getUpdateScript():
+            return False
+        if not self.isUpdateDeterministic():
+            return True
+        return rehash() != oldHash
+
     def _getSandboxVariantId(self):
         return bytes.fromhex(self.__data['sandboxVariantId'])