Merge branch 'refactoring-a-lot-v2'

Merge version 0.2.1 in master for release. This release contains a huge refactoring of the deployment core.

Author: jonathanslenders

README.md

@@ -0,0 +1,22 @@
+0. implement __dir__
+2. Add dummy text backend to telnet_auth.
+3. clean-up logging in standalone shell.
+5. write documentation (sphinx)
+6. correct exceptions.
+7. Take $TERM from the client terminal. (In case of a fuse-filesystem-system, we can easily have another $TERM for each connection.)
+8. test 'hosts' vs. 'host'
+10. test exceptions in action.
+12. we should have at least a role named 'host' in SimpleNode
+17. When moving to another thread,  hostcontext should be aware of that. Guarantee thread safety.
+18. Implement walk-function
+20. implement filesystem in userspace mapping.   (sftp, etc...)
+21. update README
+22. Rename get and put to get_file and put_file and wrap it around Host.open()
+24. Inspection of required_property gives NotImplementedError, show a nicer message.
+
+25. Allow creation of Env() objects which disallow connection to hosts. By
+    using this, it's possible to safely and quickly evaluate all Query objects.
+    And Look for reverse relationsships. e.g.  List all the queries that point
+    to a certain node. "--list-references" command.
+
+https://code.google.com/p/fusepy/source/browse/trunk/sftp.py

TODO

@@ -1 +1 @@
-__version__ = '0.1.11'
+__version__ = '0.2.1'

deployer/__init__.py

@@ -13,9 +13,10 @@
 import termcolor
 import termios
 import time
+import logging
 
 from twisted.internet import fdesc
-from deployer.pty import select
+from deployer.pseudo_terminal import select
 from deployer.std import raw_mode
 from deployer.console import Console
 
@@ -107,6 +108,8 @@ def handle(self, parts):
         h = self.root
         parts = parts[:]
 
+        logging.info('Handle command line action "%s"' % ' '.join(parts))
+
         while h and parts:# and not h.is_leaf:
             try:
                 h = h.get_subhandler(parts[0])
@@ -579,7 +582,7 @@ def read(self):
                     else:
                         c = self.stdin.read(1)
 
-                    if c == '[': # (91)
+                    if c in ('[', 'O'): # (91, 68)
                         c = self.stdin.read(1)
 
                         # Cursor to left
@@ -638,17 +641,6 @@ def read(self):
                         elif c == 'F':
                             self.end()
 
-                    elif c == 'O':
-                        c = self.stdin.read(1)
-
-                        # Home
-                        if c == 'H':
-                            self.home()
-
-                        # End
-                        elif c == 'F':
-                            self.end()
-
                 # Insert character
                 else:
                     if self.vi_navigation:

deployer/cli.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+from deployer import __version__
+from deployer.contrib.default_config import example_settings
+from deployer.run.socket_client import list_sessions
+from deployer.run.socket_client import start as start_client
+from deployer.run.socket_server import start as start_server
+from deployer.run.standalone_shell import start as start_standalone
+from deployer.run.telnet_server import start as start_telnet_server
+
+import docopt
+import getopt
+import getpass
+import sys
+
+__doc__ = \
+"""Usage:
+  client.py run [-s | --single-threaded | --socket SOCKET] [--path PATH]
+                  [--non-interactive] [--log LOGFILE]
+                  [--] [ACTION PARAMETER...]
+  client.py listen [--log LOGFILE] [--non-interactive] [--socket SOCKET]
+  client.py connect (--socket SOCKET) [--path PATH] [--] [ACTION PARAMETER...]
+  client.py telnet-server [--port PORT] [--log LOGFILE] [--non-interactive]
+  client.py list-sessions
+  client.py -h | --help
+  client.py --version
+
+Options:
+  -h, --help             : Display this help text.
+  -s, --single-threaded  : Single threaded mode.
+  --path PATH            : Start the shell at the node with this location.
+  --non-interactive      : If possible, run script with as few interactions as
+                           possible.  This will always choose the default
+                           options when asked for questions.
+  --log LOGFILE          : Write logging info to this file. (For debugging.)
+  --socket SOCKET        : The path of the unix socket.
+  --version              : Show version information.
+"""
+
+def start(root_service, name=sys.argv[0], extra_loggers=None):
+    """
+    Client startup point.
+    """
+    a = docopt.docopt(__doc__.replace('client.py', name), version=__version__)
+
+    interactive = not a['--non-interactive']
+    action = a['ACTION']
+    parameters = a['PARAMETER']
+    path = a['--path'].split('.') if a['--path'] else None
+    extra_loggers = extra_loggers or []
+
+    # Socket name variable
+    # In case of integers, they map to /tmp/deployer.sock.username.X
+    socket_name = a['--socket']
+
+    if socket_name is not None and socket_name.isdigit():
+        socket_name = '/tmp/deployer.sock.%s.%s' % (getpass.getuser(), socket_name)
+
+    # List sessions
+    if a['list-sessions']:
+        list_sessions()
+
+    # Telnet server
+    elif a['telnet-server']:
+        port = int(a['PORT']) if a['PORT'] is not None else 23
+        start_telnet_server(root_service, logfile=a['--log'], port=port,
+                extra_loggers=extra_loggers)
+
+    # Socket server
+    elif a['listen']:
+        socket_name = start_server(root_service, daemonized=False,
+                    shutdown_on_last_disconnect=False,
+                    interactive=interactive, logfile=a['--log'], socket=a['--socket'],
+                    extra_loggers=extra_loggers)
+
+    # Connect to socket
+    elif a['connect']:
+        start_client(socket_name, path, action_name=action, parameters=parameters)
+
+    # Single threaded client
+    elif a['run'] and a['--single-threaded']:
+        start_standalone(root_service, interactive=interactive, cd_path=path,
+                action_name=action, parameters=parameters, logfile=a['--log'],
+                extra_loggers=extra_loggers)
+
+    # Multithreaded
+    elif a['run']:
+        # If no socket has been given. Start a daemonized server in the
+        # background, and use that socket instead.
+        if not socket_name:
+            socket_name = start_server(root_service, daemonized=True,
+                    shutdown_on_last_disconnect=True, interactive=interactive,
+                    logfile=a['--log'], extra_loggers=extra_loggers)
+
+        start_client(socket_name, path, action_name=action, parameters=parameters)
+
+
+if __name__ == '__main__':
+    start(root_service=example_settings)

deployer/client.py

@@ -4,11 +4,40 @@
 import sys
 import random
 
+__doc__ = \
+"""
+The ``console`` object is an interface for user interaction from within a
+``Node``. Among the input methods are choice lists, plain text input and password
+input.
+
+It has output methods that take the terminal size into account, like pagination
+and multi-column display. It takes care of the pseudo terminal underneat.
+
+Example:
+
+::
+
+    class MyNode(Node):
+        def do_something(self):
+            if self.console.confirm('Should we really do this?', default=True):
+                # Do it...
+                pass
+
+.. note:: When the script runs in a shell that was started with the
+    ``--non-interactive`` option, the default options will always be chosen
+    automatically.
+
+"""
+
+
 class NoInput(Exception):
     pass
 
 
 class Console(object):
+    """
+    Interface for user interaction from within a ``Node``.
+    """
     def __init__(self, pty):
         self._pty = pty
 
@@ -18,8 +47,12 @@ def is_interactive(self):
 
     def input(self, label, is_password=False, answers=None, default=None):
         """
-        Ask for input. (like raw_input, but nice colored.)
-        'answers' can be either None or a list of the accepted answers.
+        Ask for plain text input. (Similar to raw_input.)
+
+        :param is_password: Show stars instead of the actual user input.
+        :type is_password: bool
+        :param answers: A list of the accepted answers or None.
+        :param default: Default answer.
         """
         def print_question():
             answers_str = (' [%s]' % (','.join(answers)) if answers else '')
@@ -84,7 +117,10 @@ def read_answer():
 
     def choice(self, question, options, allow_random=False, default=None):
         """
-        `options`: (name, value) list
+        :param options: List of (name, value) tuples.
+        :type options: list
+        :param allow_random: If ``True``, the default option becomes 'choose random'.
+        :type allow_random: bool
         """
         if len(options) == 0:
             raise NoInput('No options given.')
@@ -126,7 +162,8 @@ def choice(self, question, options, allow_random=False, default=None):
 
     def confirm(self, question, default=None):
         """
-        Print this yes/no question, and return True when the user answers 'yes'.
+        Print this yes/no question, and return ``True`` when the user answers
+        'Yes'.
         """
         answer = 'invalid'
 
@@ -140,72 +177,85 @@ def confirm(self, question, default=None):
         return answer in ('yes', 'y')
 
     #
-    # Service selector
+    # Node selector
     #
 
-    def select_service(self, root_service, prompt='Select service', filter=None):
+    def select_node(self, root_node, prompt='Select a node', filter=None):
         """
-        Show autocompletion for service selection.
+        Show autocompletion for node selection.
         """
         from deployer.cli import ExitCLILoop, Handler, HandlerType, CLInterface
 
-        class ServiceHandler(Handler):
-            def __init__(self, service):
-                self.service = service
+        class NodeHandler(Handler):
+            def __init__(self, node):
+                self.node = node
 
             @property
             def is_leaf(self):
-                return not filter or filter(self.service)
+                return not filter or filter(self.node)
 
             @property
             def handler_type(self):
-                class ServiceType(HandlerType):
-                    color = self.service.get_group().color
-                return ServiceType()
+                class NodeType(HandlerType):
+                    color = self.node.get_group().color
+                return NodeType()
 
             def complete_subhandlers(self, part):
-                for name, subservice in self.service.get_subservices():
+                for name, subnode in self.node.get_subnodes():
                     if name.startswith(part):
-                        yield name, ServiceHandler(subservice)
+                        yield name, NodeHandler(subnode)
 
             def get_subhandler(self, name):
-                if self.service.has_subservice(name):
-                    subservice = self.service.get_subservice(name)
-                    return ServiceHandler(subservice)
+                if self.node.has_subnode(name):
+                    subnode = self.node.get_subnode(name)
+                    return NodeHandler(subnode)
 
             def __call__(self, context):
-                raise ExitCLILoop(self.service)
+                raise ExitCLILoop(self.node)
 
-        root_handler = ServiceHandler(root_service)
+        root_handler = NodeHandler(root_node)
 
         class Shell(CLInterface):
             @property
             def prompt(self):
                 return colored('\n%s > ' % prompt, 'cyan')
 
-            not_found_message = 'Service not found...'
-            not_a_leaf_message = 'Not a valid service...'
+            not_found_message = 'Node not found...'
+            not_a_leaf_message = 'Not a valid node...'
 
-        service_result = Shell(self._pty, root_handler).cmdloop()
+        node_result = Shell(self._pty, root_handler).cmdloop()
 
-        if not service_result:
+        if not node_result:
             raise NoInput
 
-        return select_service_isolation(service_result)
+        return select_node_isolation(node_result)
 
-    def select_service_isolation(self, service):
+    def select_node_isolation(self, node):
         """
         Ask for a host, from a list of hosts.
         """
-        if service._is_isolated:
-            return service
-        else:
-            options = [ (i.name, i.service) for i in service.get_isolations() ]
+        from deployer.inspection import Inspector
+        from deployer.node import IsolationIdentifierType
+
+        # List isolations first. (This is a list of index/node tuples.)
+        options = [
+                (' '.join(index), node) for index, node in
+                Inspector(node).iter_isolations(identifier_type=IsolationIdentifierType.HOSTS_SLUG)
+                ]
+
+        if len(options) > 1:
             return self.choice('Choose a host', options, allow_random=True)
+        else:
+            return options[0][1]
 
     def lesspipe(self, line_iterator):
         """
-        Paginator for output.
+        Paginator for output. This will print one page at a time. When the user
+        presses a key, the next page is printed. ``Ctrl-c`` or ``q`` will quit
+        the paginator.
+
+        :param line_iterator: A generator function that yields lines (without
+                              trailing newline)
         """
         height = self._pty.get_size()[0] - 1
 
@@ -242,8 +292,8 @@ def lesspipe(self, line_iterator):
 
     def in_columns(self, item_iterator, margin_left=0):
         """
-        `item_iterator' should be an iterable, which yields either
-        basestring, or (colored_item, length)
+        :param item_iterator: An iterable, which yields either ``basestring``
+                              instances, or (colored_item, length) tuples.
         """
         # Helper functions for extracting items from the iterator
         def get_length(item):