Type sig for @generate

Author: spookylukey

src/parsy/__init__.py

@@ -9,7 +9,7 @@
 import re
 from dataclasses import dataclass
 from functools import reduce, wraps
-from typing import Any, Callable, FrozenSet, Generic, Optional, TypeVar, Union
+from typing import Any, Callable, FrozenSet, Generator, Generic, Optional, TypeVar, Union
 
 
 from .version import __version__  # noqa: F401
@@ -359,11 +359,25 @@ def __lshift__(self: Parser[OUT1], other: Parser) -> Parser[OUT1]:
         return self.skip(other)
 
 
-# combinator syntax
-def generate(fn):
+# TODO:
+# I think @generate is unfixable. It's not surprising, because
+# we are doing something genuninely unusual with generator functions.
+
+# The return value of a `@generate` parser is now OK.
+
+# But we have no type checking within a user's @generate function.
+
+# The big issue is that each `val = yield parser` inside a @generate parser has
+# a different type, and we'd like those to be typed checked. But the
+# `Generator[...]` expects a homogeneous stream of yield and send types,
+# whereas we have pairs of yield/send types which need to match within the
+# pair, but each pair can be completely different from the next in the stream
+
+
+def generate(fn: Callable[[], Generator[Parser[Any], Any, OUT]]) -> Parser[OUT]:
     @Parser
     @wraps(fn)
-    def generated(stream, index):
+    def generated(stream: str, index: int) -> Result[OUT]:
         # start up the generator
         iterator = fn()
 
@@ -379,9 +393,6 @@ def generated(stream, index):
                 index = result.index
         except StopIteration as stop:
             returnVal = stop.value
-            if isinstance(returnVal, Parser):
-                return returnVal(stream, index).aggregate(result)
-
             return Result.success(index, returnVal).aggregate(result)
 
     return generated

tests/test_parsy.py

@@ -4,11 +4,13 @@
 except ImportError:
     enum = None
 import re
+from typing import Generator
 import unittest
-from collections import namedtuple
-from datetime import date
+
+from typing import Any
 
 from parsy import (
+    Parser,
     ParseError,
     any_char,
     char_from,
@@ -134,7 +136,7 @@ def test_generate(self):
         x = y = None
 
         @generate
-        def xy():
+        def xy() -> Generator[Parser[Any], Any, int]:
             nonlocal x
             nonlocal y
             x = yield string("x")
@@ -145,14 +147,6 @@ def xy():
         self.assertEqual(x, "x")
         self.assertEqual(y, "y")
 
-    def test_generate_return_parser(self):
-        @generate
-        def example():
-            yield string("x")
-            return string("y")
-
-        self.assertEqual(example.parse("xy"), "y")
-
     def test_mark(self):
         parser = (letter.many().mark() << string("\n")).many()
 
@@ -189,7 +183,7 @@ def test_multiple_failures(self):
 
     def test_generate_backtracking(self):
         @generate
-        def xy():
+        def xy() -> Generator[Parser[Any], Any, None]:
             yield string("x")
             yield string("y")
             assert False
@@ -480,7 +474,7 @@ def test_decimal_digit(self):
 
     def test_line_info(self):
         @generate
-        def foo():
+        def foo() -> Generator[Any, Any, tuple[str, tuple[int, int]]]:
             i = yield line_info
             l = yield any_char
             return (l, i)