Do not rely on CMake to find Python libraries

Author: micbou

.travis.yml

@@ -47,20 +47,12 @@ addons:
      #   https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json
      - ubuntu-toolchain-r-test  # for new libstdc++
      - llvm-toolchain-precise-3.7  # for clang
-     - deadsnakes  # for various versions of python
-     - kalakris-cmake # for a more recent version of cmake (needed for ninja-build)
+     - george-edison55-precise-backports # for a more recent version of cmake (3.2.3)
     packages:
+     - cmake-data
      - cmake
      - clang-3.7
      - ninja-build
-     # The confusing part is that on Travis Linux with YCMD_PYTHON_VERSION=3.3,
-     # we build the C++ parts against the below system python3.3, but run
-     # against the pyenv python3.3. This is because stupid cmake 2.8.11 has a
-     # bug preventing it from finding the pyenv pythons (ostensibly; I haven't
-     # checked, but online reports say the issue is gone with cmake 3.4).
-     # Everything still works though, it's just weird.
-     - python3.3
-     - python3.3-dev
      # Everything below is a Python build dep (though it depends on Python
      # version). We need them because pyenv builds Python.
      - libssl-dev

build.py

@@ -7,12 +7,17 @@
 from __future__ import division
 from __future__ import absolute_import
 
+from shutil import rmtree
+from tempfile import mkdtemp
+import errno
+import re
+import multiprocessing
 import os
-import subprocess
 import os.path as p
-import sys
+import platform
 import shlex
-import errno
+import subprocess
+import sys
 
 PY_MAJOR, PY_MINOR = sys.version_info[ 0 : 2 ]
 if not ( ( PY_MAJOR == 2 and PY_MINOR >= 6 ) or
@@ -33,11 +38,22 @@
 
 sys.path.insert( 1, p.abspath( p.join( DIR_OF_THIRD_PARTY, 'argparse' ) ) )
 
-from tempfile import mkdtemp
-from shutil import rmtree
-import platform
 import argparse
-import multiprocessing
+
+NO_DYNAMIC_PYTHON_ERROR = (
+  'ERROR: found static Python library ({library}) but a dynamic one is '
+  'required. You must use a Python compiled with the {flag} flag. '
+  'If using pyenv, you need to run the command:\n'
+  '  export PYTHON_CONFIGURE_OPTS="{flag}"\n'
+  'before installing a Python version.' )
+NO_PYTHON_LIBRARY_ERROR = 'ERROR: unable to find an appropriate Python library.'
+
+LIBRARY_LDCONFIG_REGEX = re.compile(
+  '(?P<library>\S+) \(.*\) => (?P<path>\S+)' )
+
+
+def OnLinux():
+  return platform.system() == 'Linux'
 
 
 def OnMac():
@@ -120,82 +136,125 @@ def CheckOutput( *popen_args, **kwargs ):
   return output
 
 
+def GetPythonNameOnUnix():
+  python_name = 'python' + str( PY_MAJOR ) + '.' + str( PY_MINOR )
+  # Python 3 has an 'm' suffix on Unix platforms, for instance libpython3.3m.so.
+  if PY_MAJOR == 3:
+    python_name += 'm'
+  return python_name
+
+
+def GetStandardPythonLocationsOnUnix( prefix, name ):
+  return ( '{0}/lib/lib{1}'.format( prefix, name ),
+           '{0}/include/{1}'.format( prefix, name ) )
+
+
+def FindPythonLibrariesOnLinux():
+  python_name = GetPythonNameOnUnix()
+  python_library_root, python_include = GetStandardPythonLocationsOnUnix(
+    sys.exec_prefix, python_name )
+
+  python_library = python_library_root + '.so'
+  if p.isfile( python_library ):
+    return python_library, python_include
+
+  python_library = python_library_root + '.a'
+  if p.isfile( python_library ):
+    sys.exit( NO_DYNAMIC_PYTHON_ERROR.format( library = python_library,
+                                              flag = '--enable-shared' ) )
+
+  # On some distributions (Ubuntu for instance), the Python system library is
+  # not installed in its default path: /usr/lib. We use the ldconfig tool to
+  # find it.
+  python_library = 'lib' + python_name + '.so'
+  ldconfig_output = CheckOutput( [ 'ldconfig', '-p' ] ).strip().decode( 'utf8' )
+  for line in ldconfig_output.splitlines():
+    match = LIBRARY_LDCONFIG_REGEX.search( line )
+    if match and match.group( 'library' ) == python_library:
+      return match.group( 'path' ), python_include
+
+  sys.exit( NO_PYTHON_LIBRARY_ERROR )
+
+
+def FindPythonLibrariesOnMac():
+  python_prefix = sys.exec_prefix
+
+  python_library = p.join( python_prefix, 'Python' )
+  if p.isfile( python_library ):
+    return python_library, p.join( python_prefix, 'Headers' )
+
+  python_name = GetPythonNameOnUnix()
+  python_library_root, python_include = GetStandardPythonLocationsOnUnix(
+    python_prefix, python_name )
+
+  # On MacOS, ycmd does not work with statically linked python library.
+  # It typically manifests with the following error when there is a
+  # self-compiled python without --enable-framework (or, technically
+  # --enable-shared):
+  #
+  #   Fatal Python error: PyThreadState_Get: no current thread
+  #
+  # The most likely explanation for this is that both the ycm_core.so and the
+  # python binary include copies of libpython.a (or whatever included
+  # objects). When the python interpreter starts it initializes only the
+  # globals within its copy, so when ycm_core.so's copy starts executing, it
+  # points at its own copy which is uninitialized.
+  #
+  # Some platforms' dynamic linkers (ld.so) are able to resolve this when
+  # loading shared libraries at runtime[citation needed], but OSX seemingly
+  # cannot.
+  #
+  # So we do 2 things special on OS X:
+  #  - look for a .dylib first
+  #  - if we find a .a, raise an error.
+  python_library = python_library_root + '.dylib'
+  if p.isfile( python_library ):
+    return python_library, python_include
+
+  python_library = python_library_root + '.a'
+  if p.isfile( python_library ):
+    sys.exit( NO_DYNAMIC_PYTHON_ERROR.format( library = python_library,
+                                              flag = '--enable-framework' ) )
+
+  sys.exit( NO_PYTHON_LIBRARY_ERROR )
+
+
+def FindPythonLibrariesOnWindows():
+  python_prefix = sys.exec_prefix
+  python_name = 'python' + str( PY_MAJOR ) + str( PY_MINOR )
+
+  python_library = p.join( python_prefix, 'libs', python_name + '.lib' )
+  if p.isfile( python_library ):
+    return python_library, p.join( python_prefix, 'include' )
+
+  sys.exit( NO_PYTHON_LIBRARY_ERROR )
+
+
+def FindPythonLibraries():
+  if OnLinux():
+    return FindPythonLibrariesOnLinux()
+
+  if OnMac():
+    return FindPythonLibrariesOnMac()
+
+  if OnWindows():
+    return FindPythonLibrariesOnWindows()
+
+  sys.exit( 'ERROR: your platform is not supported by this script. Follow the '
+            'Full Installation Guide instructions in the documentation.' )
+
+
 def CustomPythonCmakeArgs():
   # The CMake 'FindPythonLibs' Module does not work properly.
   # So we are forced to do its job for it.
+  print( 'Searching Python {major}.{minor} libraries...'.format(
+    major = PY_MAJOR, minor = PY_MINOR ) )
 
-  print( 'Searching for python libraries...' )
-
-  python_prefix = CheckOutput( [
-    'python-config',
-    '--prefix'
-  ] ).strip().decode( 'utf8' )
-
-  if p.isfile( p.join( python_prefix, '/Python' ) ):
-    python_library = p.join( python_prefix, '/Python' )
-    python_include = p.join( python_prefix, '/Headers' )
-    print( 'Using OSX-style libs from {0}'.format( python_prefix ) )
-  else:
-    major_minor = CheckOutput( [
-      'python',
-      '-c',
-      'import sys;i=sys.version_info;print( "%d.%d" % (i[0], i[1]) )'
-    ] ).strip().decode( 'utf8' )
-    which_python = 'python' + major_minor
-
-    # Python 3 has an 'm' suffix, for instance libpython3.3m.a
-    if major_minor.startswith( '3' ):
-      which_python += 'm'
-
-    lib_python = '{0}/lib/lib{1}'.format( python_prefix, which_python ).strip()
-
-    print( 'Searching for python with prefix: {0} and lib {1}:'.format(
-      python_prefix, which_python ) )
-
-    # On MacOS, ycmd does not work with statically linked python library.
-    # It typically manifests with the following error when there is a
-    # self-compiled python without --enable-framework (or, technically
-    # --enable-shared):
-    #
-    #   Fatal Python error: PyThreadState_Get: no current thread
-    #
-    # The most likely explanation for this is that both the ycm_core.so and the
-    # python binary include copies of libpython.a (or whatever included
-    # objects). When the python interpreter starts it initializes only the
-    # globals within its copy, so when ycm_core.so's copy starts executing, it
-    # points at its own copy which is uninitialized.
-    #
-    # Some platforms' dynamic linkers (ld.so) are able to resolve this when
-    # loading shared libraries at runtime[citation needed], but OSX seemingly
-    # cannot.
-    #
-    # So we do 2 things special on OS X:
-    #  - look for a .dylib first
-    #  - if we find a .a, raise an error.
-
-    if p.isfile( '{0}.dylib'.format( lib_python ) ):
-      python_library = '{0}.dylib'.format( lib_python )
-    elif p.isfile( '/usr/lib/lib{0}.dylib'.format( which_python ) ):
-      # For no clear reason, python2.6 only exists in /usr/lib on OS X and
-      # not in the python prefix location
-      python_library = '/usr/lib/lib{0}.dylib'.format( which_python )
-    elif p.isfile( '{0}.a'.format( lib_python ) ):
-      if OnMac():
-        sys.exit( 'ERROR: You must use a python compiled with '
-                  '--enable-shared or --enable-framework (and thus a {0}.dylib '
-                  'library) on OS X'.format( lib_python ) )
-
-      python_library = '{0}.a'.format( lib_python )
-    # This check is for CYGWIN
-    elif p.isfile( '{0}.dll.a'.format( lib_python ) ):
-      python_library = '{0}.dll.a'.format( lib_python )
-    else:
-      sys.exit( 'ERROR: Unable to find an appropriate python library' )
+  python_library, python_include = FindPythonLibraries()
 
-    python_include = '{0}/include/{1}'.format( python_prefix, which_python )
+  print( 'Found Python library: {0}'.format( python_library ) )
+  print( 'Found Python headers folder: {0}'.format( python_include ) )
 
-  print( 'Using PYTHON_LIBRARY={0} PYTHON_INCLUDE_DIR={1}'.format(
-      python_library, python_include ) )
   return [
     '-DPYTHON_LIBRARY={0}'.format( python_library ),
     '-DPYTHON_INCLUDE_DIR={0}'.format( python_include )
@@ -322,8 +381,7 @@ def BuildYcmdLib( args ):
 
   try:
     full_cmake_args = [ '-G', GetGenerator( args ) ]
-    if OnMac():
-      full_cmake_args.extend( CustomPythonCmakeArgs() )
+    full_cmake_args.extend( CustomPythonCmakeArgs() )
     full_cmake_args.extend( GetCmakeArgs( args ) )
     full_cmake_args.append( p.join( DIR_OF_THIS_SCRIPT, 'cpp' ) )
 

ci/appveyor/appveyor_install.bat

@@ -33,13 +33,6 @@ if %python% == 27 (
   set PYTHONHOME=%python_path%
 )
 
-:: When using Python 3 on AppVeyor, CMake will always pick the 64 bits
-:: libraries. We specifically tell CMake the right path to the libraries
-:: according to the architecture.
-if %python% == 35 (
-  set EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=%python_path%\libs\python%python%.lib"
-)
-
 appveyor DownloadFile https://bootstrap.pypa.io/get-pip.py
 python get-pip.py
 pip install -r test_requirements.txt

ci/travis/travis_install.linux.sh

@@ -17,3 +17,6 @@ ln -s /usr/bin/clang-3.7 ${HOME}/bin/gcc
 
 export PATH=${HOME}/bin:${PATH}
 
+# In order to work with ycmd, python *must* be built as a shared library. This
+# is set via the PYTHON_CONFIGURE_OPTS option.
+export PYTHON_CONFIGURE_OPTS="--enable-shared"

ci/travis/travis_install.osx.sh

@@ -15,13 +15,13 @@ REQUIREMENTS="node.js
               pkg-config
               openssl"
 
-# Install node, go, ninja, pyenv and dependencies
+# Install node, go, ninja, pyenv and dependencies.
 for pkg in $REQUIREMENTS; do
-  # Install package, or upgrade it if it is already installed
+  # Install package, or upgrade it if it is already installed.
   brew install $pkg || brew outdated $pkg || brew upgrade $pkg
 done
 
 # In order to work with ycmd, python *must* be built as a shared library. The
 # most compatible way to do this on OS X is with --enable-framework. This is
-# set via the PYTHON_CONFIGURE_OPTS option
+# set via the PYTHON_CONFIGURE_OPTS option.
 export PYTHON_CONFIGURE_OPTS="--enable-framework"

cpp/CMakeLists.txt

@@ -15,18 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
 
-if ( DEFINED ENV{TRAVIS} )
-  # We lie to Travis CI about the min CMake version we need. For what we use
-  # Travis for, the old version of CMake that it has is good enough.
-  cmake_minimum_required( VERSION 2.8 )
+if ( APPLE )
+  # OSX requires CMake >= 2.8.12, see YCM issue #1439
+  cmake_minimum_required( VERSION 2.8.12 )
 else()
-  if ( APPLE )
-    # OSX requires CMake >= 2.8.12, see YCM issue #1439
-    cmake_minimum_required( VERSION 2.8.12 )
-  else()
-    # CMake 2.8.11 is the latest available version on RHEL/CentOS 7
-    cmake_minimum_required( VERSION 2.8.11 )
-  endif()
+  # CMake 2.8.11 is the latest available version on RHEL/CentOS 7
+  cmake_minimum_required( VERSION 2.8.11 )
 endif()
 
 project( YouCompleteMe )

cpp/ycm/CMakeLists.txt

@@ -276,10 +276,15 @@ if ( EXTERNAL_LIBCLANG_PATH OR USE_SYSTEM_LIBCLANG )
   else()
     # For Macs, we do things differently; look further in this file.
     if ( NOT APPLE )
-      # Setting this to true makes sure that libraries we build will have our rpath
-      # set even without having to do "make install"
+      # Setting this to true makes sure that libraries we build will have our
+      # rpath set even without having to do "make install"
       set( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )
       set( CMAKE_INSTALL_RPATH "\$ORIGIN" )
+      # Add directories from all libraries outside the build tree to the rpath.
+      # This makes the dynamic linker able to find non system libraries that
+      # our libraries require, in particular the Python one (from pyenv for
+      # instance).
+      set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
     endif()
   endif()