Add support for multiple workspaces (palantir#601)

Author: andfoy

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
     steps:
       - checkout
       - run: pip install -e .[all] .[test]
-      - run: py.test test/
+      - run: py.test -v test/
       - run: pylint pyls test
       - run: pycodestyle pyls test
       - run: pyflakes pyls test
@@ -18,7 +18,7 @@ jobs:
     steps:
       - checkout
       - run: pip install -e .[all] .[test]
-      - run: py.test test/
+      - run: py.test -v test/
 
   lint:
     docker:

.pylintrc

@@ -15,7 +15,8 @@ disable =
     protected-access,
     too-few-public-methods,
     too-many-arguments,
-    too-many-instance-attributes
+    too-many-instance-attributes,
+    import-error
 
 [REPORTS]
 

pyls/_utils.py

@@ -3,8 +3,16 @@
 import inspect
 import logging
 import os
+import sys
 import threading
 
+PY2 = sys.version_info.major == 2
+
+if PY2:
+    import pathlib2 as pathlib
+else:
+    import pathlib
+
 log = logging.getLogger(__name__)
 
 
@@ -71,6 +79,30 @@ def find_parents(root, path, names):
     return []
 
 
+def match_uri_to_workspace(uri, workspaces):
+    if uri is None:
+        return None
+    max_len, chosen_workspace = -1, None
+    path = pathlib.Path(uri).parts
+    for workspace in workspaces:
+        try:
+            workspace_parts = pathlib.Path(workspace).parts
+        except TypeError:
+            # This can happen in Python2 if 'value' is a subclass of string
+            workspace_parts = pathlib.Path(unicode(workspace)).parts
+        if len(workspace_parts) > len(path):
+            continue
+        match_len = 0
+        for workspace_part, path_part in zip(workspace_parts, path):
+            if workspace_part == path_part:
+                match_len += 1
+        if match_len > 0:
+            if match_len > max_len:
+                max_len = match_len
+                chosen_workspace = workspace
+    return chosen_workspace
+
+
 def list_to_string(value):
     return ",".join(value) if isinstance(value, list) else value
 

pyls/python_ls.py

@@ -75,6 +75,9 @@ class PythonLanguageServer(MethodDispatcher):
     def __init__(self, rx, tx, check_parent_process=False):
         self.workspace = None
         self.config = None
+        self.root_uri = None
+        self.workspaces = {}
+        self.uri_workspace_mapper = {}
 
         self._jsonrpc_stream_reader = JsonRpcStreamReader(rx)
         self._jsonrpc_stream_writer = JsonRpcStreamWriter(tx)
@@ -115,11 +118,16 @@ def m_exit(self, **_kwargs):
         self._jsonrpc_stream_reader.close()
         self._jsonrpc_stream_writer.close()
 
+    def _match_uri_to_workspace(self, uri):
+        workspace_uri = _utils.match_uri_to_workspace(uri, self.workspaces)
+        return self.workspaces.get(workspace_uri, self.workspace)
+
     def _hook(self, hook_name, doc_uri=None, **kwargs):
         """Calls hook_name and returns a list of results from all registered handlers"""
-        doc = self.workspace.get_document(doc_uri) if doc_uri else None
+        workspace = self._match_uri_to_workspace(doc_uri)
+        doc = workspace.get_document(doc_uri) if doc_uri else None
         hook_handlers = self.config.plugin_manager.subset_hook_caller(hook_name, self.config.disabled_plugins)
-        return hook_handlers(config=self.config, workspace=self.workspace, document=doc, **kwargs)
+        return hook_handlers(config=self.config, workspace=workspace, document=doc, **kwargs)
 
     def capabilities(self):
         server_capabilities = {
@@ -152,6 +160,12 @@ def capabilities(self):
                 },
                 'openClose': True,
             },
+            'workspace': {
+                'workspaceFolders': {
+                    'supported': True,
+                    'changeNotifications': True
+                }
+            },
             'experimental': merge(self._hook('pyls_experimental_capabilities'))
         }
         log.info('Server capabilities: %s', server_capabilities)
@@ -162,7 +176,10 @@ def m_initialize(self, processId=None, rootUri=None, rootPath=None, initializati
         if rootUri is None:
             rootUri = uris.from_fs_path(rootPath) if rootPath is not None else ''
 
+        self.workspaces.pop(self.root_uri, None)
+        self.root_uri = rootUri
         self.workspace = Workspace(rootUri, self._endpoint)
+        self.workspaces[rootUri] = self.workspace
         self.config = config.Config(rootUri, initializationOptions or {},
                                     processId, _kwargs.get('capabilities', {}))
         self._dispatchers = self._hook('pyls_dispatchers')
@@ -224,8 +241,9 @@ def hover(self, doc_uri, position):
     @_utils.debounce(LINT_DEBOUNCE_S, keyed_by='doc_uri')
     def lint(self, doc_uri, is_saved):
         # Since we're debounced, the document may no longer be open
-        if doc_uri in self.workspace.documents:
-            self.workspace.publish_diagnostics(
+        workspace = self._match_uri_to_workspace(doc_uri)
+        if doc_uri in workspace.documents:
+            workspace.publish_diagnostics(
                 doc_uri,
                 flatten(self._hook('pyls_lint', doc_uri, is_saved=is_saved))
             )
@@ -243,16 +261,19 @@ def signature_help(self, doc_uri, position):
         return self._hook('pyls_signature_help', doc_uri, position=position)
 
     def m_text_document__did_close(self, textDocument=None, **_kwargs):
-        self.workspace.rm_document(textDocument['uri'])
+        workspace = self._match_uri_to_workspace(textDocument['uri'])
+        workspace.rm_document(textDocument['uri'])
 
     def m_text_document__did_open(self, textDocument=None, **_kwargs):
-        self.workspace.put_document(textDocument['uri'], textDocument['text'], version=textDocument.get('version'))
+        workspace = self._match_uri_to_workspace(textDocument['uri'])
+        workspace.put_document(textDocument['uri'], textDocument['text'], version=textDocument.get('version'))
         self._hook('pyls_document_did_open', textDocument['uri'])
         self.lint(textDocument['uri'], is_saved=True)
 
     def m_text_document__did_change(self, contentChanges=None, textDocument=None, **_kwargs):
+        workspace = self._match_uri_to_workspace(textDocument['uri'])
         for change in contentChanges:
-            self.workspace.update_document(
+            workspace.update_document(
                 textDocument['uri'],
                 change,
                 version=textDocument.get('version')
@@ -303,8 +324,27 @@ def m_text_document__signature_help(self, textDocument=None, position=None, **_k
 
     def m_workspace__did_change_configuration(self, settings=None):
         self.config.update((settings or {}).get('pyls', {}))
-        for doc_uri in self.workspace.documents:
-            self.lint(doc_uri, is_saved=False)
+        for workspace_uri in self.workspaces:
+            workspace = self.workspaces[workspace_uri]
+            for doc_uri in workspace.documents:
+                self.lint(doc_uri, is_saved=False)
+
+    def m_workspace__did_change_workspace_folders(self, added=None, removed=None, **_kwargs):
+        for removed_info in removed:
+            removed_uri = removed_info['uri']
+            self.workspaces.pop(removed_uri)
+
+        for added_info in added:
+            added_uri = added_info['uri']
+            self.workspaces[added_uri] = Workspace(added_uri, self._endpoint)
+
+        # Migrate documents that are on the root workspace and have a better
+        # match now
+        doc_uris = list(self.workspace._docs.keys())
+        for uri in doc_uris:
+            doc = self.workspace._docs.pop(uri)
+            new_workspace = self._match_uri_to_workspace(uri)
+            new_workspace._docs[uri] = doc
 
     def m_workspace__did_change_watched_files(self, changes=None, **_kwargs):
         changed_py_files = set()
@@ -321,10 +361,12 @@ def m_workspace__did_change_watched_files(self, changes=None, **_kwargs):
             # Only externally changed python files and lint configs may result in changed diagnostics.
             return
 
-        for doc_uri in self.workspace.documents:
-            # Changes in doc_uri are already handled by m_text_document__did_save
-            if doc_uri not in changed_py_files:
-                self.lint(doc_uri, is_saved=False)
+        for workspace_uri in self.workspaces:
+            workspace = self.workspaces[workspace_uri]
+            for doc_uri in workspace.documents:
+                # Changes in doc_uri are already handled by m_text_document__did_save
+                if doc_uri not in changed_py_files:
+                    self.lint(doc_uri, is_saved=False)
 
     def m_workspace__execute_command(self, command=None, arguments=None):
         return self.execute_command(command, arguments)

test/test_workspace.py

@@ -1,10 +1,24 @@
 # Copyright 2017 Palantir Technologies, Inc.
 import os
+import os.path as osp
+import sys
 from pyls import uris
 
+PY2 = sys.version_info.major == 2
+
+if PY2:
+    import pathlib2 as pathlib
+else:
+    import pathlib
+
+
 DOC_URI = uris.from_fs_path(__file__)
 
 
+def path_as_uri(path):
+    return pathlib.Path(osp.abspath(path)).as_uri()
+
+
 def test_local(pyls):
     """ Since the workspace points to the test directory """
     assert pyls.workspace.is_local()
@@ -48,3 +62,47 @@ def test_non_root_project(pyls):
     pyls.workspace.put_document(test_uri, 'assert True')
     test_doc = pyls.workspace.get_document(test_uri)
     assert project_root in test_doc.sys_path()
+
+
+def test_multiple_workspaces(tmpdir, pyls):
+    workspace1_dir = tmpdir.mkdir('workspace1')
+    workspace2_dir = tmpdir.mkdir('workspace2')
+    file1 = workspace1_dir.join('file1.py')
+    file2 = workspace2_dir.join('file1.py')
+    file1.write('import os')
+    file2.write('import sys')
+
+    msg = {
+        'uri': path_as_uri(str(file1)),
+        'version': 1,
+        'text': 'import os'
+    }
+
+    pyls.m_text_document__did_open(textDocument=msg)
+    assert msg['uri'] in pyls.workspace._docs
+
+    added_workspaces = [{'uri': path_as_uri(str(x))}
+                        for x in (workspace1_dir, workspace2_dir)]
+    pyls.m_workspace__did_change_workspace_folders(
+        added=added_workspaces, removed=[])
+
+    for workspace in added_workspaces:
+        assert workspace['uri'] in pyls.workspaces
+
+    workspace1_uri = added_workspaces[0]['uri']
+    assert msg['uri'] not in pyls.workspace._docs
+    assert msg['uri'] in pyls.workspaces[workspace1_uri]._docs
+
+    msg = {
+        'uri': path_as_uri(str(file2)),
+        'version': 1,
+        'text': 'import sys'
+    }
+    pyls.m_text_document__did_open(textDocument=msg)
+
+    workspace2_uri = added_workspaces[1]['uri']
+    assert msg['uri'] in pyls.workspaces[workspace2_uri]._docs
+
+    pyls.m_workspace__did_change_workspace_folders(
+        added=[], removed=[added_workspaces[0]])
+    assert workspace1_uri not in pyls.workspaces