Improve __builtins__.pytd

Small improvements to solver test cases.
Allow multiple matches for method call (with __any_object__).
Remove some potential causes of non-determinism (still haven't found them all).
Started finding missing/wrong stuff in pytree.pytd, but it's not finished.

Author: pludemann

builtins/__builtin__.pytd

@@ -305,7 +305,7 @@ def add(x : int, y : int) -> int
                       ["int", "bool"])
 
   def testComplexFunction(self):
-    """Test parsing of a function with unions, noneable etc."""
+    """Test parsing of a function with unions, none-able etc."""
 
     canonical = textwrap.dedent("""
         def foo(a: int, b: int or float or None, c: Foo and `s.Bar` and Zot) -> int raises Bad

parse/ast_test.py

@@ -33,7 +33,7 @@ def _FindBuiltinFile(name):
 _cached_builtins = {}
 
 
-def GetBuiltins(stdlib=True):
+def GetBuiltins(stdlib=True, builtin_name="__builtin__"):
   """Get the "default" AST used to lookup built in types.
 
   Get an AST for all Python builtins as well as the most commonly used standard
@@ -43,6 +43,7 @@ def GetBuiltins(stdlib=True):
     stdlib: Whether to load the standard library, too. If this is False,
       TypeDeclUnit.modules will be empty. If it's True, it'll contain modules
       like itertools and signal.
+    builtin_name: The base part of the builtins file name.
 
   Returns:
     A pytd.TypeDeclUnit instance. It'll directly contain the builtin classes
@@ -57,7 +58,8 @@ def GetBuiltins(stdlib=True):
   # We use the same parser instance to parse all builtin files. This changes
   # the run time from 1.0423s to 0.5938s (for 21 builtins).
   p = parser.TypeDeclParser()
-  builtins = p.Parse(_FindBuiltinFile("__builtin__.pytd"), name="__builtin__")
+  builtins = p.Parse(
+      _FindBuiltinFile(builtin_name + ".pytd"), name=builtin_name)
   # We list modules explicitly, because we might have to extract them out of
   # a PAR file, which doesn't have good support for listing directories.
   modules = ["array", "codecs", "errno", "fcntl", "gc", "itertools", "marshal",

parse/builtins.py

@@ -718,28 +718,14 @@ def p_type_and(self, p):
     #              "type1 and type2" would be useful for anything. We
     #              should remove it.
     # This rule depends on precedence specification
-    if (isinstance(p[1], pytd.IntersectionType) and
-        isinstance(p[3], pytd.NamedType)):
-      p[0] = pytd.IntersectionType(p[1].type_list + (p[3],))
-    elif (isinstance(p[1], pytd.NamedType) and
-          isinstance(p[3], pytd.IntersectionType)):
-      # associative
-      p[0] = pytd.IntersectionType(((p[1],) + p[3].type_list))
-    else:
-      p[0] = pytd.IntersectionType((p[1], p[3]))
+    # IntersectionType flattens any contained IntersectinType's
+    p[0] = pytd.IntersectionType((p[1], p[3]))
 
   def p_type_or(self, p):
     """type : type OR type"""
     # This rule depends on precedence specification
-    if (isinstance(p[1], pytd.UnionType) and
-        isinstance(p[3], pytd.NamedType)):
-      p[0] = pytd.UnionType(p[1].type_list + (p[3],))
-    elif (isinstance(p[1], pytd.NamedType) and
-          isinstance(p[3], pytd.UnionType)):
-      # associative
-      p[0] = pytd.UnionType((p[1],) + p[3].type_list)
-    else:
-      p[0] = pytd.UnionType((p[1], p[3]))
+    # UnionType flattens any contained UnionType's
+    p[0] = pytd.UnionType((p[1], p[3]))
 
   # This is parameterized type
   # TODO(raoulDoc): support data types in future?

parse/parser.py

@@ -346,6 +346,10 @@ def VisitNamedType(self, node):
     raise ValueError("Unreplaced NamedType: {!s} {!r}".format(node, node))
 
   def VisitClassType(self, node):
+    # TODO: Can we give more context for this error? It's not very
+    #                  useful when it says that "T" is unresolved (e.g., from
+    #                  "def foo(x: list<T>))" ... it would be nice to know what
+    #                  it's inside.
     if node.cls is None:
       raise ValueError("Unresolved ClassType: {!s} {!r}".format(node, node))
 

parse/visitors.py

@@ -20,6 +20,7 @@
 """AST representation of a pytd file."""
 
 
+import itertools
 import re
 from pytypedecl.parse import node
 
@@ -292,10 +293,14 @@ class UnionType(node.Node('type_list')):
   __slots__ = ()
 
   # NOTE: type_list is kept as a tuple, to preserve the original order
-  #       even though in most respects it acts like a frozenset
+  #       even though in most respects it acts like a frozenset.
+  #       It also flattens the input, such that printing without
+  #       parentheses gives the same result.
 
   def __new__(cls, type_list):
-    return super(UnionType, cls).__new__(cls, tuple(type_list))
+    flattened = itertools.chain.from_iterable(
+        t.type_list if isinstance(t, UnionType) else [t] for t in type_list)
+    return super(UnionType, cls).__new__(cls, tuple(flattened))
 
   def __hash__(self):
     # See __eq__ - order doesn't matter, so use frozenset
@@ -315,8 +320,35 @@ def __ne__(self, other):
 
 # TODO: Do we still need this?
 class IntersectionType(node.Node('type_list')):
+  """An intersection type that contains all types in self.type_list."""
   __slots__ = ()
 
+  # NOTE: type_list is kept as a tuple, to preserve the original order
+  #       even though in most respects it acts like a frozenset.
+  #       It also flattens the input, such that printing without
+  #       parentheses gives the same result.
+
+  def __new__(cls, type_list):
+    flattened = itertools.chain.from_iterable(
+        t.type_list if isinstance(t, IntersectionType) else [t]
+        for t in type_list)
+    return super(IntersectionType, cls).__new__(cls, tuple(flattened))
+
+  def __hash__(self):
+    # See __eq__ - order doesn't matter, so use frozenset
+    return hash(frozenset(self.type_list))
+
+  def __eq__(self, other):
+    if self is other:
+      return True
+    if isinstance(other, IntersectionType):
+      # equality doesn't care about the ordering of the type_list
+      return frozenset(self.type_list) == frozenset(other.type_list)
+    return NotImplemented
+
+  def __ne__(self, other):
+    return not self == other
+
 
 class GenericType(node.Node('base_type', 'parameters')):
   """Generic type. Takes a base type and type paramters.

pytd.py

@@ -279,7 +279,7 @@ def match_type_against_type(self, t1, t2, subst):
       raise AssertionError("Don't know how to match %s against %s" % (
           type(t1), type(t2)))
 
-  def match_signature_again_signature(self, sig1, sig2, subst, skip_self):
+  def match_signature_against_signature(self, sig1, sig2, subst, skip_self):
     """Match a pytd.Signature against another pytd.Signature.
 
     Args:
@@ -318,8 +318,9 @@ def match_signature_against_function(self, sig, f, subst, skip_self=False):
     # TODO: We should abort after the first matching signature, to get
     # more precise types in the presence of overloading.
     return booleq.And(
-        booleq.Or(self.match_signature_again_signature(sig, s, subst, skip_self)
-                  for s in f.signatures)
+        booleq.Or(
+            self.match_signature_against_signature(sig, s, subst, skip_self)
+            for s in f.signatures)
         for inner_sig in sig.Visit(optimize.ExpandSignatures()))
 
   def match_function_against_function(self, f1, f2, subst, skip_self=False):