Copyright	(c) Lev Dvorkin 2022
License	MIT
Maintainer	[email protected]
Stability	Experimental
Safe Haskell	None
Language	Haskell2010

Text.Tokenizer.Types

Description

This module contains common types used by uniqueness checking and tokenizing algorithms

Synopsis

newtype Alt a = Alt [a]
data Count
- = One
- | Some
data Repeatable c = Repeatable {
- getCnt :: Count
- getBWS :: BlackWhiteSet c
}
type TokId = Int
data Token k c = Token {
- name :: k
- behind, ahead :: [BlackWhiteSet c]
- body :: [Repeatable c]
}
data RToken c = RToken {
- tokId :: TokId
- rbehind, ahead :: [Repeatable c]
- body :: [Repeatable c]
}
makeRToken :: TokId -> Token k c -> RToken c

Documentation

newtype Alt a Source #

Type synonym for list monad used as a collection of alternatives

Constructors

Alt [a]

Instances

Instances details

Monad Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods (>>=) :: Alt a -> (a -> Alt b) -> Alt b # (>>) :: Alt a -> Alt b -> Alt b # return :: a -> Alt a #
Functor Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods fmap :: (a -> b) -> Alt a -> Alt b # (<$) :: a -> Alt b -> Alt a #
Applicative Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods pure :: a -> Alt a # (<>) :: Alt (a -> b) -> Alt a -> Alt b # liftA2 :: (a -> b -> c) -> Alt a -> Alt b -> Alt c # (>) :: Alt a -> Alt b -> Alt b # (<*) :: Alt a -> Alt b -> Alt a #
Foldable Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods fold :: Monoid m => Alt m -> m # foldMap :: Monoid m => (a -> m) -> Alt a -> m # foldMap' :: Monoid m => (a -> m) -> Alt a -> m # foldr :: (a -> b -> b) -> b -> Alt a -> b # foldr' :: (a -> b -> b) -> b -> Alt a -> b # foldl :: (b -> a -> b) -> b -> Alt a -> b # foldl' :: (b -> a -> b) -> b -> Alt a -> b # foldr1 :: (a -> a -> a) -> Alt a -> a # foldl1 :: (a -> a -> a) -> Alt a -> a # toList :: Alt a -> [a] # null :: Alt a -> Bool # length :: Alt a -> Int # elem :: Eq a => a -> Alt a -> Bool # maximum :: Ord a => Alt a -> a # minimum :: Ord a => Alt a -> a # sum :: Num a => Alt a -> a # product :: Num a => Alt a -> a #
Traversable Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods traverse :: Applicative f => (a -> f b) -> Alt a -> f (Alt b) # sequenceA :: Applicative f => Alt (f a) -> f (Alt a) # mapM :: Monad m => (a -> m b) -> Alt a -> m (Alt b) # sequence :: Monad m => Alt (m a) -> m (Alt a) #
Alternative Alt Source #
Instance details Defined in Text.Tokenizer.Types Methods empty :: Alt a # (<\|>) :: Alt a -> Alt a -> Alt a # some :: Alt a -> Alt [a] # many :: Alt a -> Alt [a] #
Eq a => Eq (Alt a) Source #
Instance details Defined in Text.Tokenizer.Types Methods (==) :: Alt a -> Alt a -> Bool # (/=) :: Alt a -> Alt a -> Bool #
Ord a => Ord (Alt a) Source #
Instance details Defined in Text.Tokenizer.Types Methods compare :: Alt a -> Alt a -> Ordering # (<) :: Alt a -> Alt a -> Bool # (<=) :: Alt a -> Alt a -> Bool # (>) :: Alt a -> Alt a -> Bool # (>=) :: Alt a -> Alt a -> Bool # max :: Alt a -> Alt a -> Alt a # min :: Alt a -> Alt a -> Alt a #
Show a => Show (Alt a) Source #
Instance details Defined in Text.Tokenizer.Types Methods showsPrec :: Int -> Alt a -> ShowS # show :: Alt a -> String # showList :: [Alt a] -> ShowS #

data Count Source #

Number of symbols acceptable by Repeatable

Constructors

One
Some

Instances

Instances details

Eq Count Source #
Instance details Defined in Text.Tokenizer.Types Methods (==) :: Count -> Count -> Bool # (/=) :: Count -> Count -> Bool #
Ord Count Source #
Instance details Defined in Text.Tokenizer.Types Methods compare :: Count -> Count -> Ordering # (<) :: Count -> Count -> Bool # (<=) :: Count -> Count -> Bool # (>) :: Count -> Count -> Bool # (>=) :: Count -> Count -> Bool # max :: Count -> Count -> Count # min :: Count -> Count -> Count #
Show Count Source #
Instance details Defined in Text.Tokenizer.Types Methods showsPrec :: Int -> Count -> ShowS # show :: Count -> String # showList :: [Count] -> ShowS #

data Repeatable c Source #

BlackWhiteSet that can be repeated.

Constructors

Repeatable
Fields getCnt :: Count getBWS :: BlackWhiteSet c

Instances

Instances details

Eq c => Eq (Repeatable c) Source #
Instance details Defined in Text.Tokenizer.Types Methods (==) :: Repeatable c -> Repeatable c -> Bool # (/=) :: Repeatable c -> Repeatable c -> Bool #
Ord c => Ord (Repeatable c) Source #
Instance details Defined in Text.Tokenizer.Types Methods compare :: Repeatable c -> Repeatable c -> Ordering # (<) :: Repeatable c -> Repeatable c -> Bool # (<=) :: Repeatable c -> Repeatable c -> Bool # (>) :: Repeatable c -> Repeatable c -> Bool # (>=) :: Repeatable c -> Repeatable c -> Bool # max :: Repeatable c -> Repeatable c -> Repeatable c # min :: Repeatable c -> Repeatable c -> Repeatable c #
Show c => Show (Repeatable c) Source #
Instance details Defined in Text.Tokenizer.Types Methods showsPrec :: Int -> Repeatable c -> ShowS # show :: Repeatable c -> String # showList :: [Repeatable c] -> ShowS #

type TokId = Int Source #

Token id type synonym.

data Token k c Source #

Token with name of type k (used for uniqueness error messages and tokenizing output) over char type c.

Constructors

Token
Fields name :: k the name of token behind, ahead :: [BlackWhiteSet c] restrictions on symbols before/after matchable part NB! they are assumed to be satisfied if there are no symbols before/after matched part respectively body :: [Repeatable c] matchable sequences of char sets with possible repetitions

Instances

Instances details

(Show k, Show c) => Show (Token k c) Source #
Instance details Defined in Text.Tokenizer.Types Methods showsPrec :: Int -> Token k c -> ShowS # show :: Token k c -> String # showList :: [Token k c] -> ShowS #

data RToken c Source #

Type for internal needs. Contains autogenerated $sel:tokId:RToken and restrictions behind token are reversed

Constructors

RToken
Fields tokId :: TokId unique token's id (generated automatically) rbehind, ahead :: [Repeatable c] constraints on symbols behind/ahead of matchable part body :: [Repeatable c] matchable part of string

Instances

Instances details

Eq (RToken c) Source #	Compares by token's id
Instance details Defined in Text.Tokenizer.Types Methods (==) :: RToken c -> RToken c -> Bool # (/=) :: RToken c -> RToken c -> Bool #
Ord (RToken c) Source #	Compares by token's id
Instance details Defined in Text.Tokenizer.Types Methods compare :: RToken c -> RToken c -> Ordering # (<) :: RToken c -> RToken c -> Bool # (<=) :: RToken c -> RToken c -> Bool # (>) :: RToken c -> RToken c -> Bool # (>=) :: RToken c -> RToken c -> Bool # max :: RToken c -> RToken c -> RToken c # min :: RToken c -> RToken c -> RToken c #
Show c => Show (RToken c) Source #
Instance details Defined in Text.Tokenizer.Types Methods showsPrec :: Int -> RToken c -> ShowS # show :: RToken c -> String # showList :: [RToken c] -> ShowS #

makeRToken :: TokId -> Token k c -> RToken c Source #

Construct an RToken from Token and its id