Merge pull request #643 from jkloetzke/resubst

Add regular expression substitution string function

Author: jkloetzke

doc/manual/configuration.rst

@@ -396,6 +396,10 @@ The following built in string functions are supported:
   undefined in the tools :ref:`configuration-recipes-provideTools` environment
   definition unless the optional ``default`` is given, which is then used
   instead.
+* ``$(resubst,pattern,replacement,text[,flags])``: Replace all matches of the
+  regular expression ``pattern`` in ``text`` with ``replacement``. Flags are
+  optional. The only currently supported flag is ``i`` to ignore case while
+  searching.
 
 The following built in string functions are additionally supported in
 :ref:`package path queries <manpage-bobpaths>`. They cannot be used in recipes

pym/bob/input.py

@@ -10,7 +10,8 @@
 from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
     auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS
 from .state import BobState
-from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression
+from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression, \
+    EXTRA_STRING_FUNS
 from .tty import InfoOnce, Warn, WarnOnce, setColorMode, setParallelTUIThreshold
 from .utils import asHexStr, joinScripts, compareVersion, binStat, \
     updateDicRecursive, hashString, getPlatformTag, getPlatformString, \
@@ -3757,6 +3758,18 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
         # Begin with root layer
         allLayers = self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None)
 
+        # Add string functions added after 1.0. We did not reserve a namespace
+        # and we better not break existing recipes.
+        collisions = set(self.__stringFunctions.keys()) & set(EXTRA_STRING_FUNS.keys())
+        if collisions:
+            for k, v in EXTRA_STRING_FUNS.items():
+                if k not in collisions:
+                    self.__stringFunctions[k] = v
+            collisions = ", ".join(sorted(collisions))
+            Warn(f"Bob internal string functions shadowed by plugins: {collisions}").warn()
+        else:
+            self.__stringFunctions.update(EXTRA_STRING_FUNS)
+
         # Parse all recipes and classes of all layers. Need to be done last
         # because only by now we have loaded all plugins.
         for layer, rootDir, scriptLanguage in allLayers:

pym/bob/stringparser.py

@@ -612,10 +612,13 @@ def funMatch(args, **options):
         else:
             raise ParseError('match only supports the ignore case flag "i"')
 
-    if re.search(args[1],args[0],flags):
-        return "true"
-    else:
-        return "false"
+    try:
+        if re.search(args[1],args[0],flags):
+            return "true"
+        else:
+            return "false"
+    except re.error as e:
+        raise ParseError("Invalid $(match) regex '{}': {}".format(e.pattern, e))
 
 def funIfThenElse(args, **options):
     if len(args) != 3: raise ParseError("if-then-else expects three arguments")
@@ -682,6 +685,27 @@ def funMatchScm(args, **options):
 
     return "false"
 
+def funResubst(args, **options):
+    try:
+        [3, 4].index(len(args))
+    except ValueError:
+        raise ParseError("$(resubst) expects either three or four arguments")
+
+    flags = 0
+    if len(args) == 4:
+        if args[3] == 'i':
+            flags = re.IGNORECASE
+        else:
+            raise ParseError('$(resubst) only supports the ignore case flag "i"')
+
+    try:
+        return re.sub(args[0], args[1], args[2], flags=flags)
+    except re.error as e:
+        raise ParseError("Invalid $(resubst) regex '{}': {}".format(e.pattern, e))
+
+# Attention: do *not* add any new functions here. That will break existing
+# plugins that define a function with the same name. Use EXTRA_STRING_FUNS for
+# new functions instead.
 DEFAULT_STRING_FUNS = {
     "eq" : funEqual,
     "or" : funOr,
@@ -697,3 +721,7 @@ def funMatchScm(args, **options):
     "match" : funMatch,
     "matchScm" : funMatchScm,
 }
+
+EXTRA_STRING_FUNS = {
+    "resubst" : funResubst,
+}

test/unit/test_input_recipeset.py

@@ -5,7 +5,7 @@
 
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from unittest import TestCase
-from unittest.mock import Mock
+from unittest.mock import MagicMock, patch
 import os
 import textwrap
 import yaml
@@ -69,6 +69,14 @@ def writeAlias(self, name, content, layer=[]):
         with open(os.path.join(path, name+".yaml"), "w") as f:
             f.write(textwrap.dedent(content))
 
+    def writePlugin(self, name, content, layer=[]):
+        path = os.path.join("",
+            *(os.path.join("layers", l) for l in layer),
+            "plugins")
+        os.makedirs(path, exist_ok=True)
+        with open(os.path.join(path, name+".py"), "w") as f:
+            f.write(textwrap.dedent(content))
+
     def generate(self, sandboxEnabled=False, env={}):
         recipes = RecipeSet()
         recipes.parse(env)
@@ -2414,3 +2422,47 @@ def testSimpleDepsUpgrade(self):
         self.assertEqual(p.getBuildStep().toolDepWeak, {"tool-c"})
         self.assertEqual(p.getPackageStep().toolDep, {"tool-a", "tool-b", "tool-c"})
         self.assertEqual(p.getPackageStep().toolDepWeak, set())
+
+class TestPlugins(RecipesTmp, TestCase):
+
+    def testWarnStringFunCollision(self):
+        """A warning is issued if a plugin defines post-1.0 string function"""
+        self.writeRecipe("root", """\
+            root: True
+            privateEnvironment:
+                VAR1: "$(resubst,X,Y,AXBX)"
+                VAR2: "$(custom)"
+            packageVars: [VAR1, VAR2]
+            """)
+        self.writeConfig({
+            "plugins" : [ "funs" ]
+        })
+        self.writePlugin("funs", """\
+            def resubst(args, **options):
+                return "plugin called"
+            def custom(args, **options):
+                return "custom called"
+
+            manifest = {
+                'apiVersion' : "1.0",
+                'stringFunctions' : {
+                    "resubst" : resubst,
+                    "custom" : custom,
+                }
+            }
+            """)
+
+        warnObjMock = MagicMock()
+        warnClassMock = MagicMock(return_value=warnObjMock)
+
+        # Patch "Warn" class import at bob.input module
+        with patch('bob.input.Warn', warnClassMock):
+            p = self.generate().walkPackagePath("root")
+
+        # The parser should warn about the "resubst" collision
+        warnObjMock.warn.assert_called()
+        self.assertIn("resubst", warnClassMock.call_args.args[0])
+
+        # Make sure the plugin is called instead of the internal implementation.
+        self.assertEqual(p.getPackageStep().getEnv(),
+                         {"VAR1" : "plugin called", "VAR2" : "custom called"})

test/unit/test_input_stringparser.py

@@ -9,7 +9,7 @@
 from bob.stringparser import StringParser
 from bob.stringparser import funEqual, funNotEqual, funNot, funOr, \
     funAnd, funMatch, funIfThenElse, funSubst, funStrip, \
-    funSandboxEnabled, funToolDefined, funToolEnv
+    funSandboxEnabled, funToolDefined, funToolEnv, funResubst
 from bob.errors import ParseError
 
 def echo(args, **options):
@@ -197,6 +197,10 @@ def testMatch(self):
         self.assertRaises(ParseError, funMatch, ["a","b","x"])
         self.assertRaises(ParseError, funMatch, ["a","b","i","y"])
 
+        # broken regex
+        with self.assertRaises(ParseError):
+            funMatch(["b", r'\c'])
+
     def testIfThenElse(self):
         self.assertRaises(ParseError, funIfThenElse, ["a", "b"])
         self.assertEqual(funIfThenElse(["true", "a", "b"]), "a")
@@ -252,3 +256,23 @@ def testToolEnv(self):
         # Get real var
         self.assertEqual(funToolEnv(["foo", "bar"], __tools=tools), "baz")
         self.assertEqual(funToolEnv(["foo", "bar", "def"], __tools=tools), "baz")
+
+    def testResubst(self):
+        # Wrong number of arguments
+        with self.assertRaises(ParseError):
+            funResubst(["foo", "bar"])
+        with self.assertRaises(ParseError):
+            funResubst(["foo", "bar", "baz", "extra", "toomuch"])
+
+        # Unsupported flag
+        with self.assertRaises(ParseError):
+            funResubst(["a", "b", "abc", "%"])
+
+        # broken regex
+        with self.assertRaises(ParseError):
+            funResubst([r'\c', "b", "abc"])
+
+        self.assertEqual(funResubst(["X", "Y", "AXBXCX"]), "AYBYCY")
+        self.assertEqual(funResubst([r"\.[^.]+$", "", "1.2.3"]), "1.2")
+        self.assertEqual(funResubst(["[X]", "Y", "AXBx"]), "AYBx")
+        self.assertEqual(funResubst(["[x]", "Y", "AXBx", "i"]), "AYBY")