Don't use sys.prefix for library roots. Fixes #1422 (#1455)

src/ptvsd/__main__.py

@@ -9,7 +9,7 @@
 import runpy
 import sys
 import site
-
+import sysconfig
 
 # ptvsd can also be invoked directly rather than via -m. In this case, the
 # first entry on sys.path is the one added automatically by Python for the
@@ -31,19 +31,19 @@
 # ptvsd with that in effect, and then remove it before continuing execution.
 if __name__ == '__main__' and 'ptvsd' not in sys.modules:
     sys.path[0] = os.path.dirname(sys.path[0])
-    import ptvsd # noqa
+    import ptvsd  # noqa
     del sys.path[0]
 
-
 import pydevd
 
+import threading  # Import after pydevd.
+
 import ptvsd._remote
 import ptvsd.options
 import ptvsd.runner
 import ptvsd.version
 from ptvsd.multiproc import listen_for_subprocesses
 
-
 # When forming the command line involving __main__.py, it might be tempting to
 # import it as a module, and then use its __file__. However, that does not work
 # reliably, because __file__ can be a relative path - and when it is relative,
@@ -54,7 +54,6 @@
 # absolute path at import time.
 __file__ = os.path.abspath(__file__)
 
-
 TARGET = '<filename> | -m <module> | -c <code> | --pid <pid>'
 
 HELP = ('''ptvsd %s
@@ -73,17 +72,21 @@ def string(s):
         s = s.decode(sys.getfilesystemencoding())
     return s
 
+
 def in_range(parser, start, stop):
+
     def parse(s):
         n = parser(s)
         if start is not None and n < start:
             raise ValueError('must be >= %s' % start)
         if stop is not None and n >= stop:
             raise ValueError('must be < %s' % stop)
         return n
+
     return parse
 
-port = in_range(int, 0, 2**16)
+
+port = in_range(int, 0, 2 ** 16)
 
 pid = in_range(int, 0, None)
 
@@ -92,27 +95,38 @@ def print_help_and_exit(switch, it):
     print(HELP, file=sys.stderr)
     sys.exit(0)
 
+
 def print_version_and_exit(switch, it):
     print(ptvsd.version.__version__)
     sys.exit(0)
 
+
 def set_arg(varname, parser):
+
     def action(arg, it):
         value = parser(next(it))
         setattr(ptvsd.options, varname, value)
+
     return action
 
+
 def set_true(varname):
+
     def do(arg, it):
         setattr(ptvsd.options, varname, True)
+
     return do
 
+
 def set_target(kind, parser=None):
+
     def do(arg, it):
         ptvsd.options.target_kind = kind
         ptvsd.options.target = arg if parser is None else parser(next(it))
+
     return do
 
+
 def set_nodebug(arg, it):
     # --nodebug implies --client
     ptvsd.options.no_debug = True
@@ -124,26 +138,26 @@ def set_nodebug(arg, it):
     # ======                    ===========         ======                                  ========
 
     # Switches that are documented for use by end users.
-    (('-?', '-h', '--help'),    None,               print_help_and_exit,                    False),
-    (('-V', '--version'),       None,               print_version_and_exit,                 False),
-    ('--host',                  '<address>',        set_arg('host', string),                True),
-    ('--port',                  '<port>',           set_arg('port', port),                  False),
-    ('--wait',                  None,               set_true('wait'),                       False),
-    ('--multiprocess',          None,               set_true('multiprocess'),               False),
-    ('--log-dir',               '<path>',           set_arg('log_dir', string),             False),
+    (('-?', '-h', '--help'), None, print_help_and_exit, False),
+    (('-V', '--version'), None, print_version_and_exit, False),
+    ('--host', '<address>', set_arg('host', string), True),
+    ('--port', '<port>', set_arg('port', port), False),
+    ('--wait', None, set_true('wait'), False),
+    ('--multiprocess', None, set_true('multiprocess'), False),
+    ('--log-dir', '<path>', set_arg('log_dir', string), False),
 
     # Switches that are used internally by the IDE or ptvsd itself.
-    ('--nodebug',               None,               set_nodebug,                            False),
-    ('--client',                None,               set_true('client'),                     False),
-    ('--subprocess-of',         '<pid>',            set_arg('subprocess_of', pid),          False),
-    ('--subprocess-notify',     '<port>',           set_arg('subprocess_notify', port),     False),
+    ('--nodebug', None, set_nodebug, False),
+    ('--client', None, set_true('client'), False),
+    ('--subprocess-of', '<pid>', set_arg('subprocess_of', pid), False),
+    ('--subprocess-notify', '<port>', set_arg('subprocess_notify', port), False),
 
     # Targets. The '' entry corresponds to positional command line arguments,
     # i.e. the ones not preceded by any switch name.
-    ('',                        '<filename>',       set_target('file'),                     False),
-    ('-m',                      '<module>',         set_target('module', string),           False),
-    ('-c',                      '<code>',           set_target('code', string),             False),
-    ('--pid',                   '<pid>',            set_target('pid', pid),                 False),
+    ('', '<filename>', set_target('file'), False),
+    ('-m', '<module>', set_target('module', string), False),
+    ('-c', '<code>', set_target('code', string), False),
+    ('--pid', '<pid>', set_target('pid', pid), False),
 ]
 
 
@@ -212,6 +226,13 @@ def setup_connection():
         if os.path.exists(path) and os.path.basename(path) == 'site-packages':
             ptvsd.log.debug('Folder with "site-packages" in sys.path: {0}', path)
 
+    for path_name in {'stdlib', 'platstdlib', 'purelib', 'platlib'} & set(
+            sysconfig.get_path_names()):
+        ptvsd.log.debug('sysconfig {0}: {1}', path_name, sysconfig.get_path(path_name))
+
+    ptvsd.log.debug('os dir: {0}', os.path.dirname(os.__file__))
+    ptvsd.log.debug('threading dir: {0}', os.path.dirname(threading.__file__))
+
     opts = ptvsd.options
     pydevd.apply_debugger_options({
         'server': not opts.client,
@@ -328,13 +349,16 @@ def run_code():
 
 
 def attach_to_pid():
+
     def quoted_str(s):
         if s is None:
             return s
         assert not isinstance(s, bytes)
         unescaped = set(chr(ch) for ch in range(32, 127)) - {'"', "'", '\\'}
+
         def escape(ch):
             return ch if ch in unescaped else '\\u%04X' % ord(ch)
+
         return 'u"' + ''.join(map(escape, s)) + '"'
 
     ptvsd.log.info('Attaching to process with ID {0}', ptvsd.options.target)

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_filtering.py

@@ -7,6 +7,7 @@
 import pydevd_file_utils
 import json
 from collections import namedtuple
+from _pydev_imps._pydev_saved_modules import threading
 
 try:
     xrange  # noqa
@@ -147,11 +148,21 @@ def __init__(self):
     def _get_default_library_roots(cls):
         # Provide sensible defaults if not in env vars.
         import site
-        roots = [sys.prefix]
-        if hasattr(sys, 'base_prefix'):
-            roots.append(sys.base_prefix)
-        if hasattr(sys, 'real_prefix'):
-            roots.append(sys.real_prefix)
+
+        roots = []
+
+        try:
+            import sysconfig  # Python 2.7 onwards only.
+        except ImportError:
+            pass
+        else:
+            for path_name in set(('stdlib', 'platstdlib', 'purelib', 'platlib')) & set(sysconfig.get_path_names()):
+                roots.append(sysconfig.get_path(path_name))
+
+        # Make sure we always get at least the standard library location (based on the `os` and
+        # `threading` modules -- it's a bit weird that it may be different on the ci, but it happens).
+        roots.append(os.path.dirname(os.__file__))
+        roots.append(os.path.dirname(threading.__file__))
 
         if hasattr(site, 'getusersitepackages'):
             site_paths = site.getusersitepackages()
@@ -173,6 +184,8 @@ def _get_default_library_roots(cls):
             if os.path.exists(path) and os.path.basename(path) == 'site-packages':
                 roots.append(path)
 
+        roots.extend([os.path.realpath(path) for path in roots])
+
         return sorted(set(roots))
 
     def _normpath(self, filename):

tests/conftest.py

@@ -51,24 +51,42 @@ def pytest_report_header(config):
     else:
         print('Number of processors: %s' % (multiprocessing.cpu_count(),))
 
+    import sysconfig
+    from os.path import realpath
+
     print('Relevant system paths:')
-    print('sys.prefix: %s' % (sys.prefix,))
+    print('sys.prefix: %s (%s)' % (sys.prefix, realpath(sys.prefix)))
 
     if hasattr(sys, 'base_prefix'):
-        print('sys.base_prefix: %s' % (sys.base_prefix,))
+        print('sys.base_prefix: %s (%s)' % (
+            sys.base_prefix, realpath(sys.base_prefix)))
 
     if hasattr(sys, 'real_prefix'):
-        print('sys.real_prefix: %s' % (sys.real_prefix,))
+        print('sys.real_prefix: %s (%s)' % (
+            sys.real_prefix, realpath(sys.real_prefix)))
 
     if hasattr(site, 'getusersitepackages'):
-        print('site.getusersitepackages(): %s' % (site.getusersitepackages(),))
+        print('site.getusersitepackages(): %s (%s)' % (
+            site.getusersitepackages(), realpath(site.getusersitepackages())))
 
     if hasattr(site, 'getsitepackages'):
-        print('site.getsitepackages(): %s' % (site.getsitepackages(),))
+        print('site.getsitepackages(): %s (%s)' % (
+            site.getsitepackages(), realpath(site.getsitepackages())))
 
     for path in sys.path:
         if os.path.exists(path) and os.path.basename(path) == 'site-packages':
-            print('Folder with "site-packages" in sys.path: %s' % (path,))
+            print('Folder with "site-packages" in sys.path: %s (%s)' % (
+                path, realpath(path)))
+
+    for path_name in sorted(sysconfig.get_path_names()):
+        print('sysconfig: %s: %s (%s)' % (
+            path_name, sysconfig.get_path(path_name), realpath(sysconfig.get_path(path_name))))
+
+    print('os module dir: %s (%s)' % (
+        os.path.dirname(os.__file__), realpath(os.path.dirname(os.__file__))))
+
+    print('threading module dir: %s (%s)' % (
+        os.path.dirname(threading.__file__), realpath(os.path.dirname(threading.__file__))))
 
 
 @pytest.hookimpl(hookwrapper=True, tryfirst=True)