Network.HTTP.Robots

Synopsis

• type Robot = [([UserAgent], [Directive])]
• data UserAgent
  • = Wildcard
  • | Literal ByteString
• type Path = ByteString
• data Directive
  • = Allow Path
  • | Disallow Path
  • | CrawlDelay Int
• parseRobots :: ByteString -> Either String Robot
• robotP :: Parser Robot
• skipSpace :: Parser ()
• directiveP :: Parser Directive
• agentP :: Parser UserAgent
• commentsP :: Parser ()
• tokenP :: Parser ByteString
• canAccess :: ByteString -> Robot -> Path -> Bool

Documentation