From 0bab02d6252f004256b41df3e5e077ee8de79abf Mon Sep 17 00:00:00 2001 From: bageljr Date: Sat, 2 Apr 2022 14:56:07 -0500 Subject: [PATCH 01/62] switch to sqllite3 --- rope/contrib/autoimport.py | 165 +++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 61 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 4d6890e2a..ea821b42e 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -1,13 +1,12 @@ +import pathlib import re +import sqlite3 +from os import listdir +from pkgutil import walk_packages +from typing import Dict -from rope.base import builtins -from rope.base import exceptions -from rope.base import libutils -from rope.base import pynames -from rope.base import pyobjects -from rope.base import resources -from rope.base import resourceobserver -from rope.base import taskhandle +from rope.base import (builtins, exceptions, libutils, pynames, pyobjects, + resourceobserver, resources, taskhandle) from rope.refactor import importutils @@ -19,6 +18,8 @@ class AutoImport(object): """ + packages: Dict[str, Dict[str, str]] = {} + def __init__(self, project, observe=True, underlined=False): """Construct an AutoImport object @@ -29,10 +30,9 @@ def __init__(self, project, observe=True, underlined=False): """ self.project = project self.underlined = underlined - self.names = project.data_files.read_data("globalnames") - if self.names is None: - self.names = {} - project.data_files.add_write_hook(self._write) + self.connection = sqlite3.connect(f"{project.ropefolder.path}/autoimport.db") + self.connection.execute("create table if not exists names(name, module)") + self._check_all() # XXX: using a filtered observer observer = resourceobserver.ResourceObserver( changed=self._changed, moved=self._moved, removed=self._removed @@ -40,44 +40,60 @@ def __init__(self, project, observe=True, underlined=False): if observe: project.add_observer(observer) + def _check_import(self, module): + """ + Checks the ability to import an external package, removes it if not avalible + """ + # Not Implemented Yet, silently will fail + pass + + def _check_all(self): + """ + Checks all modules and removes bad ones + """ + pass + def import_assist(self, starting): """Return a list of ``(name, module)`` tuples This function tries to find modules that have a global name that starts with `starting`. """ - # XXX: breaking if gave up! use generators - result = [] - for module in self.names: - for global_name in self.names[module]: - if global_name.startswith(starting): - result.append((global_name, module)) - return result + results = self.connection.execute( + "select (name, module) from name where name like (?)", (starting,) + ) + for result in results: + if not self._check_import(result(1)): + del results[result] + return results def get_modules(self, name): """Return the list of modules that have global `name`""" - result = [] - for module in self.names: - if name in self.names[module]: - result.append(module) - return result + results = self.connection.execute( + "SELECT module FROM names WHERE name LIKE (?)", (name,) + ).fetchall() + for result in results: + if not self._check_import(result(0)): + del results[result] + return results def get_all_names(self): """Return the list of all cached global names""" - result = set() - for module in self.names: - result.update(set(self.names[module])) - return result + self._check_all() + results = self.connection.execute( + "select module from names where name" + ).fetchall() + return results - def get_name_locations(self, name): + def get_name_locations(self, target_name): """Return a list of ``(resource, lineno)`` tuples""" result = [] - for module in self.names: - if name in self.names[module]: + for name, module in self.connection.execute("select (name, module) "): + if target_name in name: try: pymodule = self.project.get_module(module) - if name in pymodule: - pyname = pymodule[name] + if target_name in pymodule: + pyname = pymodule[target_name] module, lineno = pyname.get_definition_location() if module is not None: resource = module.get_module().get_resource() @@ -107,23 +123,44 @@ def generate_cache( self.update_resource(file, underlined) job_set.finished_job() + def _handle_import_error(self, *args): + pass + def generate_modules_cache( - self, modules, underlined=None, task_handle=taskhandle.NullTaskHandle() + self, modules=None, underlined=None, task_handle=taskhandle.NullTaskHandle() ): """Generate global name cache for modules listed in `modules`""" job_set = task_handle.create_jobset( - "Generating autoimport cache for modules", len(modules) + "Generating autoimport cache for modules", + "all" if modules is None else len(modules), ) - for modname in modules: - job_set.started_job("Working on <%s>" % modname) - if modname.endswith(".*"): - mod = self.project.find_module(modname[:-2]) - if mod: - for sub in submodules(mod): - self.update_resource(sub, underlined) - else: - self.update_module(modname, underlined) - job_set.finished_job() + if modules is None: + folders = self.project.get_python_path_folders() + for package in walk_packages(onerror=self._handle_import_error): + self._generate_module_cache( + f"{package.name}", + job_set, + underlined, + task_handle, + ) + + else: + for modname in modules: + self._generate_module_cache(modname, job_set, underlined, task_handle) + + def _generate_module_cache( + self, modname, job_set, underlined=None, task_handle=taskhandle.NullTaskHandle() + ): + job_set.started_job("Working on <%s>" % modname) + if modname.endswith(".*"): + # This is wildly inneffecient given that we know the path already + mod = self.project.find_module(modname[:-2]) + if mod: + for sub in submodules(mod): + self.update_resource(sub, underlined) + else: + self.update_module(modname, underlined) + job_set.finished_job() def clear_cache(self): """Clear all entries in global-name cache @@ -132,7 +169,7 @@ def clear_cache(self): regenerating global names. """ - self.names.clear() + self.connection.execute("drop table names") def find_insertion_line(self, code): """Guess at what line the new import should be inserted""" @@ -158,6 +195,7 @@ def update_resource(self, resource, underlined=None): pymodule = self.project.get_pymodule(resource) modname = self._module_name(resource) self._add_names(pymodule, modname, underlined) + except exceptions.ModuleSyntaxError: pass @@ -167,6 +205,10 @@ def update_module(self, modname, underlined=None): `modname` is the name of a module. """ try: + if self.connection.execute( + "select count(1) from names where module is (?)", (modname,) + ): + return pymodule = self.project.get_module(modname) self._add_names(pymodule, modname, underlined) except exceptions.ModuleNotFoundError: @@ -178,22 +220,19 @@ def _module_name(self, resource): def _add_names(self, pymodule, modname, underlined): if underlined is None: underlined = self.underlined - globals = [] if isinstance(pymodule, pyobjects.PyDefinedObject): attributes = pymodule._get_structural_attributes() else: attributes = pymodule.get_attributes() for name, pyname in attributes.items(): - if not underlined and name.startswith("_"): - continue - if isinstance(pyname, (pynames.AssignedName, pynames.DefinedName)): - globals.append(name) - if isinstance(pymodule, builtins.BuiltinModule): - globals.append(name) - self.names[modname] = globals - - def _write(self): - self.project.data_files.write_data("globalnames", self.names) + if underlined or name.startswith("_"): + if isinstance( + pyname, + (pynames.AssignedName, pynames.DefinedName, builtins.BuiltinModule), + ): + self.connection.execute( + "insert into names(name,module) values (?,?)", (name, modname) + ) def _changed(self, resource): if not resource.is_folder(): @@ -202,15 +241,19 @@ def _changed(self, resource): def _moved(self, resource, newresource): if not resource.is_folder(): modname = self._module_name(resource) - if modname in self.names: - del self.names[modname] + self._del_if_exist(modname) self.update_resource(newresource) + def _del_if_exist(self, module_name): + self.connection.execute("delete from names where module = ?", (module_name,)) + def _removed(self, resource): if not resource.is_folder(): modname = self._module_name(resource) - if modname in self.names: - del self.names[modname] + self._del_if_exist(modname) + + def close(self): + self.connection.close() def submodules(mod): From 0df9167ca8f8922eae0eb6169783c4ea77bc5309 Mon Sep 17 00:00:00 2001 From: bageljr Date: Sun, 3 Apr 2022 23:51:11 -0500 Subject: [PATCH 02/62] use ast --- rope/contrib/autoimport.py | 264 ++++++++++++++++++++++++------------- 1 file changed, 170 insertions(+), 94 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index ea821b42e..b272fadf3 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -1,15 +1,35 @@ +import ast import pathlib import re import sqlite3 -from os import listdir -from pkgutil import walk_packages -from typing import Dict +import sys +from enum import Enum +from typing import Dict, Generator, List, Optional, Tuple from rope.base import (builtins, exceptions, libutils, pynames, pyobjects, resourceobserver, resources, taskhandle) +from rope.base.project import File, Folder from rope.refactor import importutils +class Source(Enum): + PROJECT = 0 # Obviously any project packages come first + MANUAL = 1 # Any packages manually added are probably important to the user + STANDARD = 2 # We want to favor standard library items + SITE_PACKAGE = 3 + UNKNOWN = 4 + + +def get_package_source(package: pathlib.Path) -> Source: + """Detect the source of a given package. Rudimentary implementation.""" + if package.as_posix().__contains__("site-packages"): + return Source.SITE_PACKAGE + if package.as_posix().startswith(sys.prefix): + return Source.STANDARD + else: + return Source.UNKNOWN + + class AutoImport(object): """A class for finding the module that provides a name @@ -18,9 +38,9 @@ class AutoImport(object): """ - packages: Dict[str, Dict[str, str]] = {} + connection: sqlite3.Connection - def __init__(self, project, observe=True, underlined=False): + def __init__(self, project, observe=True, underlined=False, memory=True): """Construct an AutoImport object If `observe` is `True`, listen for project changes and update @@ -30,8 +50,11 @@ def __init__(self, project, observe=True, underlined=False): """ self.project = project self.underlined = underlined - self.connection = sqlite3.connect(f"{project.ropefolder.path}/autoimport.db") - self.connection.execute("create table if not exists names(name, module)") + db_path = ":memory:" if memory else f"{project.ropefolder.path}/autoimport.db" + self.connection = sqlite3.connect(db_path) + self.connection.execute( + "create table if not exists names(name TEXT, module TEXT, package TEXT, source INTEGER)" + ) self._check_all() # XXX: using a filtered observer observer = resourceobserver.ResourceObserver( @@ -40,12 +63,12 @@ def __init__(self, project, observe=True, underlined=False): if observe: project.add_observer(observer) - def _check_import(self, module): + def _check_import(self, module) -> bool: """ Checks the ability to import an external package, removes it if not avalible """ # Not Implemented Yet, silently will fail - pass + return True def _check_all(self): """ @@ -60,48 +83,50 @@ def import_assist(self, starting): that starts with `starting`. """ results = self.connection.execute( - "select (name, module) from name where name like (?)", (starting,) - ) + "select name, module from names where name like (?)", (starting,) + ).fetchall() for result in results: - if not self._check_import(result(1)): + if not self._check_import(result[1]): del results[result] return results + def exact_match(self, target: str): + # TODO implement exact match + pass + def get_modules(self, name): """Return the list of modules that have global `name`""" results = self.connection.execute( "SELECT module FROM names WHERE name LIKE (?)", (name,) ).fetchall() for result in results: - if not self._check_import(result(0)): + if not self._check_import(result[0]): del results[result] return results def get_all_names(self): """Return the list of all cached global names""" self._check_all() - results = self.connection.execute( - "select module from names where name" - ).fetchall() + results = self.connection.execute("select name from names").fetchall() return results - def get_name_locations(self, target_name): - """Return a list of ``(resource, lineno)`` tuples""" - result = [] - for name, module in self.connection.execute("select (name, module) "): - if target_name in name: - try: - pymodule = self.project.get_module(module) - if target_name in pymodule: - pyname = pymodule[target_name] - module, lineno = pyname.get_definition_location() - if module is not None: - resource = module.get_module().get_resource() - if resource is not None and lineno is not None: - result.append((resource, lineno)) - except exceptions.ModuleNotFoundError: - pass - return result + # def get_name_locations(self, target_name): + # """Return a list of ``(resource, lineno)`` tuples""" + # result = [] + # for name, module in self.connection.execute("select (name, module) from names"): + # if target_name in name: + # try: + # pymodule = self.project.get_module(module) + # if target_name in pymodule: + # pyname = pymodule[target_name] + # module, lineno = pyname.get_definition_location() + # if module is not None: + # resource = module.get_module().get_resource() + # if resource is not None and lineno is not None: + # result.append((resource, lineno)) + # except exceptions.ModuleNotFoundError: + # pass + # return result def generate_cache( self, resources=None, underlined=None, task_handle=taskhandle.NullTaskHandle() @@ -127,7 +152,7 @@ def _handle_import_error(self, *args): pass def generate_modules_cache( - self, modules=None, underlined=None, task_handle=taskhandle.NullTaskHandle() + self, modules=None, task_handle=taskhandle.NullTaskHandle() ): """Generate global name cache for modules listed in `modules`""" job_set = task_handle.create_jobset( @@ -136,32 +161,80 @@ def generate_modules_cache( ) if modules is None: folders = self.project.get_python_path_folders() - for package in walk_packages(onerror=self._handle_import_error): - self._generate_module_cache( - f"{package.name}", - job_set, - underlined, - task_handle, - ) + for folder in folders: + for package in pathlib.Path(folder.path).iterdir(): + self._generate_module_cache( + package, + job_set, + task_handle, + ) else: for modname in modules: - self._generate_module_cache(modname, job_set, underlined, task_handle) + # TODO: need to find path + self._generate_module_cache( + modname, job_set, task_handle, package_source=Source.MANUAL + ) def _generate_module_cache( - self, modname, job_set, underlined=None, task_handle=taskhandle.NullTaskHandle() + self, + packagepath: pathlib.Path, + job_set, + task_handle=taskhandle.NullTaskHandle(), + recursive=True, + package_source: Source = None, ): - job_set.started_job("Working on <%s>" % modname) - if modname.endswith(".*"): - # This is wildly inneffecient given that we know the path already - mod = self.project.find_module(modname[:-2]) - if mod: - for sub in submodules(mod): - self.update_resource(sub, underlined) + if package_source is None: + package_source = get_package_source(packagepath) + package_name = packagepath.name + job_set.started_job("Working on <%s>" % packagepath.name) + if package_name.endswith(".egg-info"): + return + # TODO add so handling + if self.connection.execute( + "select * from names where package is (?)", (package_name,) + ).fetchone() is not None: + return + if recursive: + for sub in submodules(packagepath): + modname = ( + sub.relative_to(packagepath) + .as_posix() + .removesuffix(".py") + .replace("/", ".") + ) + if modname.__contains__("_"): + continue + modname = ( + package_name if modname == "." else package_name + "." + modname + ) + self._update_module(sub, modname, packagepath.name, package_source) else: - self.update_module(modname, underlined) + self._update_module( + packagepath, packagepath.name, packagepath.name, package_source + ) job_set.finished_job() + def _get_names( + self, + module: pathlib.Path, + modname: str, + package: str, + package_source: Source, + ) -> Generator[Tuple[str, str, str, int], None, None]: + with open(module, mode="rb") as file: + root_node = ast.parse(file.read()) + for node in ast.iter_child_nodes(root_node): + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + if not node.name.startswith("_"): + yield (node.name, modname, package, package_source.value) + + def _add_names(self, names_to_add, *args): + self.connection.executemany( + "insert into names(name,module,package,source) values (?,?,?,?)", + names_to_add(*args), + ) + def clear_cache(self): """Clear all entries in global-name cache @@ -189,50 +262,54 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - def update_resource(self, resource, underlined=None): - """Update the cache for global names in `resource`""" - try: - pymodule = self.project.get_pymodule(resource) - modname = self._module_name(resource) - self._add_names(pymodule, modname, underlined) - - except exceptions.ModuleSyntaxError: - pass - - def update_module(self, modname, underlined=None): + # def update_resource(self, resource, underlined=None): + # """Update the cache for global names in `resource`""" + # try: + # pymodule = self.project.get_pymodule(resource) + # modname = self._module_name(resource) + # self._add_names(pymodule, modname, underlined) + # + # except exceptions.ModuleSyntaxError: + # pass + # + def _update_module( + self, + modpath: pathlib.Path, + modname: str, + package: str, + package_source: Source, + ): """Update the cache for global names in `modname` module `modname` is the name of a module. """ - try: - if self.connection.execute( - "select count(1) from names where module is (?)", (modname,) - ): - return - pymodule = self.project.get_module(modname) - self._add_names(pymodule, modname, underlined) - except exceptions.ModuleNotFoundError: - pass + # TODO use __all__ parsing if avalible + if modpath.is_dir(): + for file in modpath.glob("*.py"): + self._update_module(file, modname, package, package_source) + + else: + self._add_names(self._get_names, modpath, modname, package, package_source) def _module_name(self, resource): return libutils.modname(resource) - def _add_names(self, pymodule, modname, underlined): - if underlined is None: - underlined = self.underlined - if isinstance(pymodule, pyobjects.PyDefinedObject): - attributes = pymodule._get_structural_attributes() - else: - attributes = pymodule.get_attributes() - for name, pyname in attributes.items(): - if underlined or name.startswith("_"): - if isinstance( - pyname, - (pynames.AssignedName, pynames.DefinedName, builtins.BuiltinModule), - ): - self.connection.execute( - "insert into names(name,module) values (?,?)", (name, modname) - ) + # def _add_names(self, pymodule, modname, underlined): + # if underlined is None: + # underlined = self.underlined + # if isinstance(pymodule, pyobjects.PyDefinedObject): + # attributes = pymodule._get_structural_attributes() + # else: + # attributes = pymodule.get_attributes() + # for name, pyname in attributes.items(): + # if underlined or name.startswith("_"): + # if isinstance( + # pyname, + # (pynames.AssignedName, pynames.DefinedName, builtins.BuiltinModule), + # ): + # self.connection.execute( + # "insert into names(name,module) values (?,?)", (name, modname) + # ) def _changed(self, resource): if not resource.is_folder(): @@ -256,14 +333,13 @@ def close(self): self.connection.close() -def submodules(mod): - if isinstance(mod, resources.File): - if mod.name.endswith(".py") and mod.name != "__init__.py": - return set([mod]) - return set() - if not mod.has_child("__init__.py"): +def submodules(mod: pathlib.Path): + """Simple submodule finder that doesn't try to import anything""" + if mod.suffix == ".py" and mod.name != "__init__.py": + return set([mod]) + if not (mod / "__init__.py").exists(): return set() result = set([mod]) - for child in mod.get_children(): + for child in mod.iterdir(): result |= submodules(child) return result From 3f2c1a8a44180665e72e00a0a3aa85b332a4626e Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 00:22:08 -0500 Subject: [PATCH 03/62] refactor to use ProcessPoolExecutor --- rope/contrib/autoimport.py | 182 ++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index b272fadf3..75b2ad34e 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -3,12 +3,11 @@ import re import sqlite3 import sys +from concurrent.futures import ProcessPoolExecutor from enum import Enum -from typing import Dict, Generator, List, Optional, Tuple +from typing import Generator, List, Tuple -from rope.base import (builtins, exceptions, libutils, pynames, pyobjects, - resourceobserver, resources, taskhandle) -from rope.base.project import File, Folder +from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.refactor import importutils @@ -20,6 +19,78 @@ class Source(Enum): UNKNOWN = 4 +Name = Tuple[str, str, str, int] + + +def _get_names( + modpath: pathlib.Path, + modname: str, + package: str, + package_source: Source, +) -> List[Name]: + """Update the cache for global names in `modname` module + + `modname` is the name of a module. + """ + # TODO use __all__ parsing if avalible + if modpath.is_dir(): + names: List[Name] = [] + for file in modpath.glob("*.py"): + names.extend(_get_names(file, modname, package, package_source)) + return names + else: + return list(_get_names_from_file(modpath, modname, package, package_source)) + + +def _find_all_names_in_package( + package_path: pathlib.Path, + recursive=True, + package_source: Source = None, +) -> List[Name]: + package_name = package_path.name + if package_source is None: + package_source = get_package_source(package_path) + if package_name.endswith(".egg-info"): + return [] + # TODO add so handling + modules: List[Tuple[pathlib.Path, str]] = [] + if package_name.endswith(".py"): + stripped_name = package_name.removesuffix(".py") + modules.append((package_path, stripped_name)) + if recursive: + for sub in submodules(package_path): + modname = ( + sub.relative_to(package_path) + .as_posix() + .removesuffix(".py") + .replace("/", ".") + ) + if modname.__contains__("_"): + continue + modname = package_name if modname == "." else package_name + "." + modname + modules.append((sub, modname)) + else: + modules.append((package_path, package_name)) + result: List[Name] = [] + for module in modules: + result.extend(_get_names(module[0], module[1], package_name, package_source)) + return result + + +def _get_names_from_file( + module: pathlib.Path, + modname: str, + package: str, + package_source: Source, +) -> Generator[Name, None, None]: + with open(module, mode="rb") as file: + root_node = ast.parse(file.read()) + for node in ast.iter_child_nodes(root_node): + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + if not node.name.startswith("_"): + yield (node.name, modname, package, package_source.value) + + def get_package_source(package: pathlib.Path) -> Source: """Detect the source of a given package. Rudimentary implementation.""" if package.as_posix().__contains__("site-packages"): @@ -110,6 +181,7 @@ def get_all_names(self): results = self.connection.execute("select name from names").fetchall() return results + # def get_name_locations(self, target_name): # """Return a list of ``(resource, lineno)`` tuples""" # result = [] @@ -159,82 +231,31 @@ def generate_modules_cache( "Generating autoimport cache for modules", "all" if modules is None else len(modules), ) + packages: List[pathlib.Path] = [] if modules is None: folders = self.project.get_python_path_folders() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): - self._generate_module_cache( - package, - job_set, - task_handle, - ) + package_name = package.name + if ( + self.connection.execute( + "select * from names where package is (?)", (package_name,) + ).fetchone() + is None + ): + packages.append(package) else: for modname in modules: # TODO: need to find path - self._generate_module_cache( - modname, job_set, task_handle, package_source=Source.MANUAL + packages.append(modname) + with ProcessPoolExecutor() as exectuor: + for name_list in exectuor.map(_find_all_names_in_package, packages): + self.connection.executemany( + "insert into names(name,module,package,source) values (?,?,?,?)", + name_list, ) - def _generate_module_cache( - self, - packagepath: pathlib.Path, - job_set, - task_handle=taskhandle.NullTaskHandle(), - recursive=True, - package_source: Source = None, - ): - if package_source is None: - package_source = get_package_source(packagepath) - package_name = packagepath.name - job_set.started_job("Working on <%s>" % packagepath.name) - if package_name.endswith(".egg-info"): - return - # TODO add so handling - if self.connection.execute( - "select * from names where package is (?)", (package_name,) - ).fetchone() is not None: - return - if recursive: - for sub in submodules(packagepath): - modname = ( - sub.relative_to(packagepath) - .as_posix() - .removesuffix(".py") - .replace("/", ".") - ) - if modname.__contains__("_"): - continue - modname = ( - package_name if modname == "." else package_name + "." + modname - ) - self._update_module(sub, modname, packagepath.name, package_source) - else: - self._update_module( - packagepath, packagepath.name, packagepath.name, package_source - ) - job_set.finished_job() - - def _get_names( - self, - module: pathlib.Path, - modname: str, - package: str, - package_source: Source, - ) -> Generator[Tuple[str, str, str, int], None, None]: - with open(module, mode="rb") as file: - root_node = ast.parse(file.read()) - for node in ast.iter_child_nodes(root_node): - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): - if not node.name.startswith("_"): - yield (node.name, modname, package, package_source.value) - - def _add_names(self, names_to_add, *args): - self.connection.executemany( - "insert into names(name,module,package,source) values (?,?,?,?)", - names_to_add(*args), - ) - def clear_cache(self): """Clear all entries in global-name cache @@ -272,27 +293,6 @@ def find_insertion_line(self, code): # except exceptions.ModuleSyntaxError: # pass # - def _update_module( - self, - modpath: pathlib.Path, - modname: str, - package: str, - package_source: Source, - ): - """Update the cache for global names in `modname` module - - `modname` is the name of a module. - """ - # TODO use __all__ parsing if avalible - if modpath.is_dir(): - for file in modpath.glob("*.py"): - self._update_module(file, modname, package, package_source) - - else: - self._add_names(self._get_names, modpath, modname, package, package_source) - - def _module_name(self, resource): - return libutils.modname(resource) # def _add_names(self, pymodule, modname, underlined): # if underlined is None: From 55de3ae8f0f0d7c47ee4c49b10a849963fb10f12 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 01:03:15 -0500 Subject: [PATCH 04/62] actually somewhat working implementation --- rope/contrib/autoimport.py | 105 ++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 75b2ad34e..fefa44024 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -5,9 +5,10 @@ import sys from concurrent.futures import ProcessPoolExecutor from enum import Enum -from typing import Generator, List, Tuple +from typing import Generator, List, Optional, Set, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle +from rope.base.project import Project from rope.refactor import importutils @@ -91,8 +92,12 @@ def _get_names_from_file( yield (node.name, modname, package, package_source.value) -def get_package_source(package: pathlib.Path) -> Source: +def get_package_source( + package: pathlib.Path, project: Optional[Project] = None +) -> Source: """Detect the source of a given package. Rudimentary implementation.""" + if project is not None and package.as_posix().__contains__(project.address): + return Source.PROJECT if package.as_posix().__contains__("site-packages"): return Source.SITE_PACKAGE if package.as_posix().startswith(sys.prefix): @@ -154,12 +159,31 @@ def import_assist(self, starting): that starts with `starting`. """ results = self.connection.execute( - "select name, module from names where name like (?)", (starting,) + "select name, module from names where name like (?)%", (starting,) ).fetchall() for result in results: if not self._check_import(result[1]): del results[result] - return results + return set( + results + ) # Remove duplicates from multiple occurences of the same item + + def search(self, name) -> Set[str]: + """Searches both modules and names for an import string""" + results: List[str] = [] + for name, module in self.connection.execute( + "SELECT name, module FROM names WHERE name LIKE (?)", (name,) + ): + results.append(f"from {module} import {name}") + for module in self.connection.execute( + "Select module FROM names where module LIKE (?)", ("%." + name,) + ): + results.append(f"from {module[0].removesuffix(f'.{name}')} import {name}") + for module in self.connection.execute( + "Select module from names where module LIKE (?)", (name,) + ): + results.append(f"import {name}") + return set(results) def exact_match(self, target: str): # TODO implement exact match @@ -168,19 +192,24 @@ def exact_match(self, target: str): def get_modules(self, name): """Return the list of modules that have global `name`""" results = self.connection.execute( - "SELECT module FROM names WHERE name LIKE (?)", (name,) + "SELECT module FROM names WHERE module LIKE (?)", (name,) ).fetchall() for result in results: if not self._check_import(result[0]): del results[result] - return results + return set(*results) - def get_all_names(self): + def get_all_names(self) -> List[str]: """Return the list of all cached global names""" self._check_all() results = self.connection.execute("select name from names").fetchall() return results + def get_all(self) -> List[Name]: + """Dumps the entire database""" + self._check_all() + results = self.connection.execute("select * from names").fetchall() + return results # def get_name_locations(self, target_name): # """Return a list of ``(resource, lineno)`` tuples""" @@ -239,7 +268,8 @@ def generate_modules_cache( package_name = package.name if ( self.connection.execute( - "select * from names where package is (?)", (package_name,) + "select * from names where package LIKE (?)", + (package_name,), ).fetchone() is None ): @@ -247,14 +277,17 @@ def generate_modules_cache( else: for modname in modules: - # TODO: need to find path + # TODO: need to find path, somehow packages.append(modname) with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(_find_all_names_in_package, packages): - self.connection.executemany( - "insert into names(name,module,package,source) values (?,?,?,?)", - name_list, - ) + self._add_names(name_list) + + def _add_names(self, names: List[Name]): + self.connection.executemany( + "insert into names(name,module,package,source) values (?,?,?,?)", + names, + ) def clear_cache(self): """Clear all entries in global-name cache @@ -283,33 +316,20 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - # def update_resource(self, resource, underlined=None): - # """Update the cache for global names in `resource`""" - # try: - # pymodule = self.project.get_pymodule(resource) - # modname = self._module_name(resource) - # self._add_names(pymodule, modname, underlined) - # - # except exceptions.ModuleSyntaxError: - # pass - # - - # def _add_names(self, pymodule, modname, underlined): - # if underlined is None: - # underlined = self.underlined - # if isinstance(pymodule, pyobjects.PyDefinedObject): - # attributes = pymodule._get_structural_attributes() - # else: - # attributes = pymodule.get_attributes() - # for name, pyname in attributes.items(): - # if underlined or name.startswith("_"): - # if isinstance( - # pyname, - # (pynames.AssignedName, pynames.DefinedName, builtins.BuiltinModule), - # ): - # self.connection.execute( - # "insert into names(name,module) values (?,?)", (name, modname) - # ) + def update_resource(self, resource): + """Update the cache for global names in `resource`""" + try: + self._add_names( + [ + resource.path, + resource.path.name, + self.project.address, + Source.PROJECT.value, + ] + ) + + except exceptions.ModuleSyntaxError: + pass def _changed(self, resource): if not resource.is_folder(): @@ -317,7 +337,7 @@ def _changed(self, resource): def _moved(self, resource, newresource): if not resource.is_folder(): - modname = self._module_name(resource) + modname = libutils.modname(resource) self._del_if_exist(modname) self.update_resource(newresource) @@ -326,10 +346,11 @@ def _del_if_exist(self, module_name): def _removed(self, resource): if not resource.is_folder(): - modname = self._module_name(resource) + modname = libutils.modname(resource) self._del_if_exist(modname) def close(self): + self.connection.commit() self.connection.close() From fa945e4a629e7076dfa9f6e038edccfa4b0fce78 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 14:16:28 -0500 Subject: [PATCH 05/62] parse __all__ --- rope/contrib/autoimport.py | 81 ++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index fefa44024..c54d55b61 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -3,9 +3,10 @@ import re import sqlite3 import sys +from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from enum import Enum -from typing import Generator, List, Optional, Set, Tuple +from typing import List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project @@ -58,7 +59,7 @@ def _find_all_names_in_package( if package_name.endswith(".py"): stripped_name = package_name.removesuffix(".py") modules.append((package_path, stripped_name)) - if recursive: + elif recursive: for sub in submodules(package_path): modname = ( sub.relative_to(package_path) @@ -83,13 +84,36 @@ def _get_names_from_file( modname: str, package: str, package_source: Source, -) -> Generator[Name, None, None]: + only_all: bool = False, +) -> List[Name]: with open(module, mode="rb") as file: root_node = ast.parse(file.read()) + results: List[Name] = [] for node in ast.iter_child_nodes(root_node): - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + if isinstance(node, ast.Assign): + for target in node.targets: + try: + if target.id == "__all__": + # TODO use all somehow + all_results: List[Name] = [] + for item in node.value.elts: + all_results.append( + ( + str(item.value), + modname, + package, + package_source.value, + ) + ) + return all_results + + except AttributeError: + # TODO handle tuple assignment + pass + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and not only_all: if not node.name.startswith("_"): - yield (node.name, modname, package, package_source.value) + results.append((node.name, modname, package, package_source.value)) + return results def get_package_source( @@ -168,22 +192,26 @@ def import_assist(self, starting): results ) # Remove duplicates from multiple occurences of the same item - def search(self, name) -> Set[str]: + def search(self, name) -> List[str]: """Searches both modules and names for an import string""" - results: List[str] = [] - for name, module in self.connection.execute( - "SELECT name, module FROM names WHERE name LIKE (?)", (name,) + results: List[Tuple[str, int]] = [] + for name, module, source in self.connection.execute( + "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) ): - results.append(f"from {module} import {name}") - for module in self.connection.execute( - "Select module FROM names where module LIKE (?)", ("%." + name,) + results.append((f"from {module} import {name}", source)) + for module, source in self.connection.execute( + "Select module, source FROM names where module LIKE (?)", ("%." + name,) ): - results.append(f"from {module[0].removesuffix(f'.{name}')} import {name}") - for module in self.connection.execute( - "Select module from names where module LIKE (?)", (name,) + results.append( + (f"from {module.removesuffix(f'.{name}')} import {name}", source) + ) + for module, source in self.connection.execute( + "Select module, source from names where module LIKE (?)", (name,) ): - results.append(f"import {name}") - return set(results) + results.append((f"import {name}", source)) + results.sort(key=lambda y: y[1]) + results_sorted = list(zip(*results))[0] + return list(OrderedDict.fromkeys(results_sorted)) def exact_match(self, target: str): # TODO implement exact match @@ -211,24 +239,6 @@ def get_all(self) -> List[Name]: results = self.connection.execute("select * from names").fetchall() return results - # def get_name_locations(self, target_name): - # """Return a list of ``(resource, lineno)`` tuples""" - # result = [] - # for name, module in self.connection.execute("select (name, module) from names"): - # if target_name in name: - # try: - # pymodule = self.project.get_module(module) - # if target_name in pymodule: - # pyname = pymodule[target_name] - # module, lineno = pyname.get_definition_location() - # if module is not None: - # resource = module.get_module().get_resource() - # if resource is not None and lineno is not None: - # result.append((resource, lineno)) - # except exceptions.ModuleNotFoundError: - # pass - # return result - def generate_cache( self, resources=None, underlined=None, task_handle=taskhandle.NullTaskHandle() ): @@ -282,6 +292,7 @@ def generate_modules_cache( with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(_find_all_names_in_package, packages): self._add_names(name_list) + self.connection.commit() def _add_names(self, names: List[Name]): self.connection.executemany( From 0a20abda664ad7500117672a00c385d91f38c607 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 14:26:06 -0500 Subject: [PATCH 06/62] improve parsing of modules with submodules --- rope/contrib/autoimport.py | 43 ++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index c54d55b61..26dc8ef02 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -6,7 +6,7 @@ from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from enum import Enum -from typing import List, Optional, Tuple +from typing import List, Optional, Set, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project @@ -27,7 +27,7 @@ class Source(Enum): def _get_names( modpath: pathlib.Path, modname: str, - package: str, + package_name: str, package_source: Source, ) -> List[Name]: """Update the cache for global names in `modname` module @@ -36,12 +36,30 @@ def _get_names( """ # TODO use __all__ parsing if avalible if modpath.is_dir(): - names: List[Name] = [] + names: List[Name] + if modpath / "__init__.py": + names = _get_names_from_file( + modpath / "__init__.py", + modname, + package_name, + package_source, + only_all=True, + ) + if len(names) > 0: + return names + names = [] for file in modpath.glob("*.py"): - names.extend(_get_names(file, modname, package, package_source)) + names.extend( + _get_names_from_file( + file, + modname + f".{file.name.removesuffix('.py')}", + package_name, + package_source, + ) + ) return names else: - return list(_get_names_from_file(modpath, modname, package, package_source)) + return _get_names_from_file(modpath, modname, package_name, package_source) def _find_all_names_in_package( @@ -61,6 +79,7 @@ def _find_all_names_in_package( modules.append((package_path, stripped_name)) elif recursive: for sub in submodules(package_path): + print(sub) modname = ( sub.relative_to(package_path) .as_posix() @@ -365,13 +384,11 @@ def close(self): self.connection.close() -def submodules(mod: pathlib.Path): +def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: """Simple submodule finder that doesn't try to import anything""" - if mod.suffix == ".py" and mod.name != "__init__.py": - return set([mod]) - if not (mod / "__init__.py").exists(): - return set() - result = set([mod]) - for child in mod.iterdir(): - result |= submodules(child) + result = set() + if mod.is_dir() and (mod / "__init__.py").exists(): + result.add(mod) + for child in mod.iterdir(): + result |= submodules(child) return result From 56460d5e48e53f1574ef8d53c77e6692bbf444be Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 14:56:06 -0500 Subject: [PATCH 07/62] fix resource parsing --- rope/contrib/autoimport.py | 55 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 26dc8ef02..5b706e585 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -10,6 +10,7 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project +from rope.base.resources import Resource from rope.refactor import importutils @@ -62,6 +63,18 @@ def _get_names( return _get_names_from_file(modpath, modname, package_name, package_source) +def _get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: + package_name: str = package_path.name + modname = ( + modpath.relative_to(package_path) + .as_posix() + .removesuffix(".py") + .replace("/", ".") + ) + modname = package_name if modname == "." else package_name + "." + modname + return modname + + def _find_all_names_in_package( package_path: pathlib.Path, recursive=True, @@ -72,6 +85,8 @@ def _find_all_names_in_package( package_source = get_package_source(package_path) if package_name.endswith(".egg-info"): return [] + if package_name.endswith(".so"): + return [] # TODO add so handling modules: List[Tuple[pathlib.Path, str]] = [] if package_name.endswith(".py"): @@ -79,16 +94,9 @@ def _find_all_names_in_package( modules.append((package_path, stripped_name)) elif recursive: for sub in submodules(package_path): - print(sub) - modname = ( - sub.relative_to(package_path) - .as_posix() - .removesuffix(".py") - .replace("/", ".") - ) + modname = _get_modname_from_path(sub, package_path) if modname.__contains__("_"): - continue - modname = package_name if modname == "." else package_name + "." + modname + continue # Exclude private items modules.append((sub, modname)) else: modules.append((package_path, package_name)) @@ -112,10 +120,13 @@ def _get_names_from_file( if isinstance(node, ast.Assign): for target in node.targets: try: + assert isinstance(target, ast.Name) if target.id == "__all__": # TODO use all somehow all_results: List[Name] = [] + assert isinstance(node.value, ast.List) for item in node.value.elts: + assert isinstance(item, ast.Constant) all_results.append( ( str(item.value), @@ -126,7 +137,7 @@ def _get_names_from_file( ) return all_results - except AttributeError: + except (AttributeError, AssertionError): # TODO handle tuple assignment pass if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and not only_all: @@ -258,7 +269,7 @@ def get_all(self) -> List[Name]: results = self.connection.execute("select * from names").fetchall() return results - def generate_cache( + def generate_resource_cache( self, resources=None, underlined=None, task_handle=taskhandle.NullTaskHandle() ): """Generate global name cache for project files @@ -346,20 +357,15 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - def update_resource(self, resource): + def update_resource(self, resource: Resource, underlined: List[str] = None): """Update the cache for global names in `resource`""" - try: - self._add_names( - [ - resource.path, - resource.path.name, - self.project.address, - Source.PROJECT.value, - ] - ) - - except exceptions.ModuleSyntaxError: - pass + resource_path: pathlib.Path = pathlib.Path(resource.real_path) + package_path: pathlib.Path = pathlib.Path(self.project.address) + resource_modname: str = _get_modname_from_path(resource_path, package_path) + names = _get_names_from_file( + resource_path, resource_modname, package_path.name, Source.PROJECT + ) + self._add_names(names) def _changed(self, resource): if not resource.is_folder(): @@ -392,3 +398,4 @@ def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: for child in mod.iterdir(): result |= submodules(child) return result + return result From b622adc1535bd0e5dfa2d83ba23739b7223425d1 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 15:33:39 -0500 Subject: [PATCH 08/62] add package finder and update_module --- rope/contrib/autoimport.py | 117 +++++++++++++++++++++++------ ropetest/contrib/autoimporttest.py | 34 +++++---- 2 files changed, 114 insertions(+), 37 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 5b706e585..10b304342 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -63,6 +63,28 @@ def _get_names( return _get_names_from_file(modpath, modname, package_name, package_source) +class PackageType(Enum): + STANDARD = 1 # Just a folder + COMPILED = 2 # .so module + SINGLE_FILE = 3 # a .py file + + +def _get_package_name_from_path( + package_path: pathlib.Path, +) -> Optional[Tuple[str, PackageType]]: + package_name = package_path.name + if package_name.endswith(".egg-info"): + return None + if package_name.endswith(".so"): + name = package_name.split(".")[0] + return (name, PackageType.COMPILED) + # TODO add so handling + if package_name.endswith(".py"): + stripped_name = package_name.removesuffix(".py") + return (stripped_name, PackageType.SINGLE_FILE) + return (package_name, PackageType.STANDARD) + + def _get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: package_name: str = package_path.name modname = ( @@ -80,18 +102,17 @@ def _find_all_names_in_package( recursive=True, package_source: Source = None, ) -> List[Name]: - package_name = package_path.name + package_tuple = _get_package_name_from_path(package_path) + if package_tuple is None: + return [] + package_name, package_type = package_tuple if package_source is None: package_source = get_package_source(package_path) - if package_name.endswith(".egg-info"): - return [] - if package_name.endswith(".so"): - return [] - # TODO add so handling modules: List[Tuple[pathlib.Path, str]] = [] - if package_name.endswith(".py"): - stripped_name = package_name.removesuffix(".py") - modules.append((package_path, stripped_name)) + if package_type is PackageType.SINGLE_FILE: + modules.append((package_path, package_name)) + elif package_type is PackageType.COMPILED: + return [] elif recursive: for sub in submodules(package_path): modname = _get_modname_from_path(sub, package_path) @@ -114,7 +135,10 @@ def _get_names_from_file( only_all: bool = False, ) -> List[Name]: with open(module, mode="rb") as file: - root_node = ast.parse(file.read()) + try: + root_node = ast.parse(file.read()) + except SyntaxError: + return [] results: List[Name] = [] for node in ast.iter_child_nodes(root_node): if isinstance(node, ast.Assign): @@ -160,6 +184,24 @@ def get_package_source( return Source.UNKNOWN +def _sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: + if len(results) == 0: + return [] + results.sort(key=lambda y: y[-1]) + results_sorted = list(zip(*results))[0] + return list(OrderedDict.fromkeys(results_sorted)) + + +def _sort_and_deduplicate_tuple( + results: List[Tuple[str, str, int]] +) -> List[Tuple[str, str]]: + if len(results) == 0: + return [] + results.sort(key=lambda y: y[-1]) + results_sorted = list(zip(*results))[:-1] + return list(OrderedDict.fromkeys(results_sorted)) + + class AutoImport(object): """A class for finding the module that provides a name @@ -213,12 +255,13 @@ def import_assist(self, starting): that starts with `starting`. """ results = self.connection.execute( - "select name, module from names where name like (?)%", (starting,) + "select name, module, source from names where name like (?)", + (starting + "%",), ).fetchall() for result in results: if not self._check_import(result[1]): del results[result] - return set( + return _sort_and_deduplicate_tuple( results ) # Remove duplicates from multiple occurences of the same item @@ -239,23 +282,31 @@ def search(self, name) -> List[str]: "Select module, source from names where module LIKE (?)", (name,) ): results.append((f"import {name}", source)) - results.sort(key=lambda y: y[1]) - results_sorted = list(zip(*results))[0] - return list(OrderedDict.fromkeys(results_sorted)) + return _sort_and_deduplicate(results) def exact_match(self, target: str): # TODO implement exact match pass - def get_modules(self, name): + def get_modules(self, name) -> List[str]: """Return the list of modules that have global `name`""" results = self.connection.execute( - "SELECT module FROM names WHERE module LIKE (?)", (name,) + "SELECT module, source FROM names WHERE module LIKE (?)", (name,) ).fetchall() for result in results: if not self._check_import(result[0]): del results[result] - return set(*results) + return _sort_and_deduplicate(results) + + def get_names(self, name: str) -> List[str]: + """Return the list of names that have global `name`""" + results = self.connection.execute( + "SELECT name, source FROM names WHERE name LIKE (?)", (name,) + ).fetchall() + for result in results: + if not self._check_import(result[0]): + del results[result] + return _sort_and_deduplicate(results) def get_all_names(self) -> List[str]: """Return the list of all cached global names""" @@ -292,6 +343,16 @@ def generate_resource_cache( def _handle_import_error(self, *args): pass + def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: + for folder in self.project.get_python_path_folders(): + for package in pathlib.Path(folder.path).iterdir(): + package_tuple = _get_package_name_from_path(package) + if package_tuple is None: + continue + if package_tuple[0] == package_name: + return package + return None + def generate_modules_cache( self, modules=None, task_handle=taskhandle.NullTaskHandle() ): @@ -305,7 +366,10 @@ def generate_modules_cache( folders = self.project.get_python_path_folders() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): - package_name = package.name + package_tuple = _get_package_name_from_path(package) + if package_tuple is None: + continue + package_name = package_tuple[0] if ( self.connection.execute( "select * from names where package LIKE (?)", @@ -317,18 +381,24 @@ def generate_modules_cache( else: for modname in modules: - # TODO: need to find path, somehow - packages.append(modname) + package_path = self._find_package_path(modname) + if package_path is None: + continue + packages.append(package_path) with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(_find_all_names_in_package, packages): self._add_names(name_list) self.connection.commit() + def update_module(self, module: str): + self.generate_modules_cache([module]) + def _add_names(self, names: List[Name]): self.connection.executemany( "insert into names(name,module,package,source) values (?,?,?,?)", names, ) + self.connection.commit() def clear_cache(self): """Clear all entries in global-name cache @@ -362,6 +432,10 @@ def update_resource(self, resource: Resource, underlined: List[str] = None): resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) resource_modname: str = _get_modname_from_path(resource_path, package_path) + package_tuple = _get_package_name_from_path(package_path) + if package_tuple is None: + return None + package_name = package_tuple[0] names = _get_names_from_file( resource_path, resource_modname, package_path.name, Source.PROJECT ) @@ -379,6 +453,7 @@ def _moved(self, resource, newresource): def _del_if_exist(self, module_name): self.connection.execute("delete from names where module = ?", (module_name,)) + self.connection.commit() def _removed(self, resource): if not resource.is_folder(): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 995eba6b9..d9aef462f 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -3,8 +3,8 @@ except ImportError: import unittest -from ropetest import testutils from rope.contrib import autoimport +from ropetest import testutils class AutoImportTest(unittest.TestCase): @@ -14,10 +14,11 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=False) + self.importer = autoimport.AutoImport(self.project, observe=False, memory=True) def tearDown(self): testutils.remove_project(self.project) + self.importer.close() super(AutoImportTest, self).tearDown() def test_simple_case(self): @@ -47,18 +48,18 @@ def test_excluding_imported_names(self): self.importer.update_resource(self.mod1) self.assertEqual([], self.importer.import_assist("pkg")) - def test_get_modules(self): + def test_get_names(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual(["mod1"], self.importer.get_names("myvar")) - def test_get_modules_inside_packages(self): + def test_get_names_inside_packages(self): self.mod1.write("myvar = None\n") self.mod2.write("myvar = None\n") self.importer.update_resource(self.mod1) self.importer.update_resource(self.mod2) self.assertEqual( - set(["mod1", "pkg.mod2"]), set(self.importer.get_modules("myvar")) + set(["mod1", "pkg.mod2"]), set(self.importer.get_names("myvar")) ) def test_trivial_insertion_line(self): @@ -84,22 +85,22 @@ def test_insertion_line_with_blank_lines(self): def test_empty_cache(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual(["mod1"], self.importer.get_names("myvar")) self.importer.clear_cache() - self.assertEqual([], self.importer.get_modules("myvar")) + self.assertEqual([], self.importer.get_names("myvar")) def test_not_caching_underlined_names(self): self.mod1.write("_myvar = None\n") self.importer.update_resource(self.mod1, underlined=False) - self.assertEqual([], self.importer.get_modules("_myvar")) + self.assertEqual([], self.importer.get_names("_myvar")) self.importer.update_resource(self.mod1, underlined=True) - self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) + self.assertEqual(["mod1"], self.importer.get_names("_myvar")) def test_caching_underlined_names_passing_to_the_constructor(self): importer = autoimport.AutoImport(self.project, False, True) self.mod1.write("_myvar = None\n") importer.update_resource(self.mod1) - self.assertEqual(["mod1"], importer.get_modules("_myvar")) + self.assertEqual(["mod1"], importer.get_names("_myvar")) def test_name_locations(self): self.mod1.write("myvar = None\n") @@ -118,7 +119,7 @@ def test_name_locations_with_multiple_occurrences(self): def test_handling_builtin_modules(self): self.importer.update_module("sys") - self.assertTrue("sys" in self.importer.get_modules("exit")) + self.assertTrue("sys" in self.importer.get_names("exit")) def test_submodules(self): self.assertEqual(set([self.mod1]), autoimport.submodules(self.mod1)) @@ -132,22 +133,23 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=True) + self.importer = autoimport.AutoImport(self.project, observe=True, memory=True) def tearDown(self): testutils.remove_project(self.project) + self.importer.close() super(AutoImportObservingTest, self).tearDown() def test_writing_files(self): self.mod1.write("myvar = None\n") - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual(["mod1"], self.importer.get_names("myvar")) def test_moving_files(self): self.mod1.write("myvar = None\n") self.mod1.move("mod3.py") - self.assertEqual(["mod3"], self.importer.get_modules("myvar")) + self.assertEqual(["mod3"], self.importer.get_names("myvar")) def test_removing_files(self): self.mod1.write("myvar = None\n") self.mod1.remove() - self.assertEqual([], self.importer.get_modules("myvar")) + self.assertEqual([], self.importer.get_names("myvar")) From 70e2b99ae6cca6d63ec0c2da66484feb47e1390c Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 17:19:46 -0500 Subject: [PATCH 09/62] propogate underlined more --- rope/contrib/autoimport.py | 132 ++++++++++++++++------------- ropetest/contrib/autoimporttest.py | 26 +++--- 2 files changed, 86 insertions(+), 72 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 10b304342..0a1b67fa7 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -30,6 +30,7 @@ def _get_names( modname: str, package_name: str, package_source: Source, + underlined: bool = False, ) -> List[Name]: """Update the cache for global names in `modname` module @@ -56,11 +57,14 @@ def _get_names( modname + f".{file.name.removesuffix('.py')}", package_name, package_source, + underlined=underlined, ) ) return names else: - return _get_names_from_file(modpath, modname, package_name, package_source) + return _get_names_from_file( + modpath, modname, package_name, package_source, underlined=underlined + ) class PackageType(Enum): @@ -101,6 +105,7 @@ def _find_all_names_in_package( package_path: pathlib.Path, recursive=True, package_source: Source = None, + underlined: bool = False, ) -> List[Name]: package_tuple = _get_package_name_from_path(package_path) if package_tuple is None: @@ -116,14 +121,16 @@ def _find_all_names_in_package( elif recursive: for sub in submodules(package_path): modname = _get_modname_from_path(sub, package_path) - if modname.__contains__("_"): + if underlined or modname.__contains__("_"): continue # Exclude private items modules.append((sub, modname)) else: modules.append((package_path, package_name)) result: List[Name] = [] for module in modules: - result.extend(_get_names(module[0], module[1], package_name, package_source)) + result.extend( + _get_names(module[0], module[1], package_name, package_source, underlined) + ) return result @@ -133,20 +140,23 @@ def _get_names_from_file( package: str, package_source: Source, only_all: bool = False, + underlined: bool = False, ) -> List[Name]: with open(module, mode="rb") as file: try: root_node = ast.parse(file.read()) - except SyntaxError: + except SyntaxError as e: + print(e) return [] results: List[Name] = [] for node in ast.iter_child_nodes(root_node): + node_names: List[str] = [] if isinstance(node, ast.Assign): for target in node.targets: try: assert isinstance(target, ast.Name) if target.id == "__all__": - # TODO use all somehow + # TODO add tuple handling all_results: List[Name] = [] assert isinstance(node.value, ast.List) for item in node.value.elts: @@ -160,13 +170,18 @@ def _get_names_from_file( ) ) return all_results - + else: + node_names.append(target.id) except (AttributeError, AssertionError): # TODO handle tuple assignment pass - if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and not only_all: - if not node.name.startswith("_"): - results.append((node.name, modname, package, package_source.value)) + elif isinstance(node, (ast.FunctionDef, ast.ClassDef)): + node_names = [node.name] + for node_name in node_names: + if underlined or not node_name.startswith("_"): + results.append((node_name, modname, package, package_source.value)) + if only_all: + return [] return results @@ -211,6 +226,8 @@ class AutoImport(object): """ connection: sqlite3.Connection + underlined: bool + project: Project def __init__(self, project, observe=True, underlined=False, memory=True): """Construct an AutoImport object @@ -235,18 +252,6 @@ def __init__(self, project, observe=True, underlined=False, memory=True): if observe: project.add_observer(observer) - def _check_import(self, module) -> bool: - """ - Checks the ability to import an external package, removes it if not avalible - """ - # Not Implemented Yet, silently will fail - return True - - def _check_all(self): - """ - Checks all modules and removes bad ones - """ - pass def import_assist(self, starting): """Return a list of ``(name, module)`` tuples @@ -284,24 +289,10 @@ def search(self, name) -> List[str]: results.append((f"import {name}", source)) return _sort_and_deduplicate(results) - def exact_match(self, target: str): - # TODO implement exact match - pass - def get_modules(self, name) -> List[str]: """Return the list of modules that have global `name`""" results = self.connection.execute( - "SELECT module, source FROM names WHERE module LIKE (?)", (name,) - ).fetchall() - for result in results: - if not self._check_import(result[0]): - del results[result] - return _sort_and_deduplicate(results) - - def get_names(self, name: str) -> List[str]: - """Return the list of names that have global `name`""" - results = self.connection.execute( - "SELECT name, source FROM names WHERE name LIKE (?)", (name,) + "SELECT module, source FROM names WHERE name LIKE (?)", (name,) ).fetchall() for result in results: if not self._check_import(result[0]): @@ -321,7 +312,10 @@ def get_all(self) -> List[Name]: return results def generate_resource_cache( - self, resources=None, underlined=None, task_handle=taskhandle.NullTaskHandle() + self, + resources=None, + underlined: bool = False, + task_handle=taskhandle.NullTaskHandle(), ): """Generate global name cache for project files @@ -340,21 +334,11 @@ def generate_resource_cache( self.update_resource(file, underlined) job_set.finished_job() - def _handle_import_error(self, *args): - pass - - def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: - for folder in self.project.get_python_path_folders(): - for package in pathlib.Path(folder.path).iterdir(): - package_tuple = _get_package_name_from_path(package) - if package_tuple is None: - continue - if package_tuple[0] == package_name: - return package - return None def generate_modules_cache( - self, modules=None, task_handle=taskhandle.NullTaskHandle() + self, + modules=None, + task_handle=taskhandle.NullTaskHandle(), ): """Generate global name cache for modules listed in `modules`""" job_set = task_handle.create_jobset( @@ -393,12 +377,9 @@ def generate_modules_cache( def update_module(self, module: str): self.generate_modules_cache([module]) - def _add_names(self, names: List[Name]): - self.connection.executemany( - "insert into names(name,module,package,source) values (?,?,?,?)", - names, - ) + def close(self): self.connection.commit() + self.connection.close() def clear_cache(self): """Clear all entries in global-name cache @@ -408,6 +389,7 @@ def clear_cache(self): """ self.connection.execute("drop table names") + self.connection.commit() def find_insertion_line(self, code): """Guess at what line the new import should be inserted""" @@ -427,18 +409,25 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - def update_resource(self, resource: Resource, underlined: List[str] = None): + def update_resource(self, resource: Resource, underlined: bool = False): """Update the cache for global names in `resource`""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) resource_modname: str = _get_modname_from_path(resource_path, package_path) package_tuple = _get_package_name_from_path(package_path) + underlined = underlined if underlined else self.underlined if package_tuple is None: return None package_name = package_tuple[0] + print((resource_path, resource_modname, package_path.name, Source.PROJECT)) names = _get_names_from_file( - resource_path, resource_modname, package_path.name, Source.PROJECT + resource_path, + resource_modname, + package_name, + Source.PROJECT, + underlined, ) + print(names) self._add_names(names) def _changed(self, resource): @@ -460,10 +449,35 @@ def _removed(self, resource): modname = libutils.modname(resource) self._del_if_exist(modname) - def close(self): + def _add_names(self, names: List[Name]): + self.connection.executemany( + "insert into names(name,module,package,source) values (?,?,?,?)", + names, + ) self.connection.commit() - self.connection.close() + def _check_import(self, module) -> bool: + """ + Checks the ability to import an external package, removes it if not avalible + """ + # Not Implemented Yet, silently will fail + return True + + def _check_all(self): + """ + Checks all modules and removes bad ones + """ + pass + + def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: + for folder in self.project.get_python_path_folders(): + for package in pathlib.Path(folder.path).iterdir(): + package_tuple = _get_package_name_from_path(package) + if package_tuple is None: + continue + if package_tuple[0] == package_name: + return package + return None def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: """Simple submodule finder that doesn't try to import anything""" diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index d9aef462f..0b9c88b60 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -48,18 +48,18 @@ def test_excluding_imported_names(self): self.importer.update_resource(self.mod1) self.assertEqual([], self.importer.import_assist("pkg")) - def test_get_names(self): + def test_get_modules(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_names("myvar")) + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) - def test_get_names_inside_packages(self): + def test_get_modules_inside_packages(self): self.mod1.write("myvar = None\n") self.mod2.write("myvar = None\n") self.importer.update_resource(self.mod1) self.importer.update_resource(self.mod2) self.assertEqual( - set(["mod1", "pkg.mod2"]), set(self.importer.get_names("myvar")) + set(["mod1", "pkg.mod2"]), set(self.importer.get_modules("myvar")) ) def test_trivial_insertion_line(self): @@ -85,22 +85,22 @@ def test_insertion_line_with_blank_lines(self): def test_empty_cache(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_names("myvar")) + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) self.importer.clear_cache() - self.assertEqual([], self.importer.get_names("myvar")) + self.assertEqual([], self.importer.get_modules("myvar")) def test_not_caching_underlined_names(self): self.mod1.write("_myvar = None\n") self.importer.update_resource(self.mod1, underlined=False) - self.assertEqual([], self.importer.get_names("_myvar")) + self.assertEqual([], self.importer.get_modules("_myvar")) self.importer.update_resource(self.mod1, underlined=True) - self.assertEqual(["mod1"], self.importer.get_names("_myvar")) + self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) def test_caching_underlined_names_passing_to_the_constructor(self): importer = autoimport.AutoImport(self.project, False, True) self.mod1.write("_myvar = None\n") importer.update_resource(self.mod1) - self.assertEqual(["mod1"], importer.get_names("_myvar")) + self.assertEqual(["mod1"], importer.get_modules("_myvar")) def test_name_locations(self): self.mod1.write("myvar = None\n") @@ -119,7 +119,7 @@ def test_name_locations_with_multiple_occurrences(self): def test_handling_builtin_modules(self): self.importer.update_module("sys") - self.assertTrue("sys" in self.importer.get_names("exit")) + self.assertTrue("sys" in self.importer.get_modules("exit")) def test_submodules(self): self.assertEqual(set([self.mod1]), autoimport.submodules(self.mod1)) @@ -142,14 +142,14 @@ def tearDown(self): def test_writing_files(self): self.mod1.write("myvar = None\n") - self.assertEqual(["mod1"], self.importer.get_names("myvar")) + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) def test_moving_files(self): self.mod1.write("myvar = None\n") self.mod1.move("mod3.py") - self.assertEqual(["mod3"], self.importer.get_names("myvar")) + self.assertEqual(["mod3"], self.importer.get_modules("myvar")) def test_removing_files(self): self.mod1.write("myvar = None\n") self.mod1.remove() - self.assertEqual([], self.importer.get_names("myvar")) + self.assertEqual([], self.importer.get_modules("myvar")) From 42ef20d191c1bc0ab57cf24c8c1ee64f6a427303 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 17:26:27 -0500 Subject: [PATCH 10/62] match public api more closely --- rope/contrib/autoimport.py | 11 +++++------ ropetest/contrib/autoimporttest.py | 26 ++++++++++++++++---------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 0a1b67fa7..e1ebd3190 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -119,7 +119,7 @@ def _find_all_names_in_package( elif package_type is PackageType.COMPILED: return [] elif recursive: - for sub in submodules(package_path): + for sub in _submodules(package_path): modname = _get_modname_from_path(sub, package_path) if underlined or modname.__contains__("_"): continue # Exclude private items @@ -252,7 +252,6 @@ def __init__(self, project, observe=True, underlined=False, memory=True): if observe: project.add_observer(observer) - def import_assist(self, starting): """Return a list of ``(name, module)`` tuples @@ -334,7 +333,6 @@ def generate_resource_cache( self.update_resource(file, underlined) job_set.finished_job() - def generate_modules_cache( self, modules=None, @@ -468,7 +466,7 @@ def _check_all(self): Checks all modules and removes bad ones """ pass - + def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: for folder in self.project.get_python_path_folders(): for package in pathlib.Path(folder.path).iterdir(): @@ -479,12 +477,13 @@ def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: return package return None -def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: + +def _submodules(mod: pathlib.Path) -> Set[pathlib.Path]: """Simple submodule finder that doesn't try to import anything""" result = set() if mod.is_dir() and (mod / "__init__.py").exists(): result.add(mod) for child in mod.iterdir(): - result |= submodules(child) + result |= _submodules(child) return result return result diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 0b9c88b60..e4d8f3071 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -11,6 +11,7 @@ class AutoImportTest(unittest.TestCase): def setUp(self): super(AutoImportTest, self).setUp() self.project = testutils.sample_project(extension_modules=["sys"]) + self.project_name = "sample_project" self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) @@ -51,7 +52,9 @@ def test_excluding_imported_names(self): def test_get_modules(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual( + [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") + ) def test_get_modules_inside_packages(self): self.mod1.write("myvar = None\n") @@ -59,7 +62,8 @@ def test_get_modules_inside_packages(self): self.importer.update_resource(self.mod1) self.importer.update_resource(self.mod2) self.assertEqual( - set(["mod1", "pkg.mod2"]), set(self.importer.get_modules("myvar")) + set([f"{self.project_name}.mod1", f"{self.project_name}.pkg.mod2"]), + set(self.importer.get_modules("myvar")), ) def test_trivial_insertion_line(self): @@ -85,7 +89,9 @@ def test_insertion_line_with_blank_lines(self): def test_empty_cache(self): self.mod1.write("myvar = None\n") self.importer.update_resource(self.mod1) - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual( + [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") + ) self.importer.clear_cache() self.assertEqual([], self.importer.get_modules("myvar")) @@ -121,10 +127,6 @@ def test_handling_builtin_modules(self): self.importer.update_module("sys") self.assertTrue("sys" in self.importer.get_modules("exit")) - def test_submodules(self): - self.assertEqual(set([self.mod1]), autoimport.submodules(self.mod1)) - self.assertEqual(set([self.mod2, self.pkg]), autoimport.submodules(self.pkg)) - class AutoImportObservingTest(unittest.TestCase): def setUp(self): @@ -134,7 +136,7 @@ def setUp(self): self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) self.importer = autoimport.AutoImport(self.project, observe=True, memory=True) - + self.project_name = "sample_project" def tearDown(self): testutils.remove_project(self.project) self.importer.close() @@ -142,12 +144,16 @@ def tearDown(self): def test_writing_files(self): self.mod1.write("myvar = None\n") - self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.assertEqual( + [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") + ) def test_moving_files(self): self.mod1.write("myvar = None\n") self.mod1.move("mod3.py") - self.assertEqual(["mod3"], self.importer.get_modules("myvar")) + self.assertEqual( + [f"{self.project_name}.mod3"], self.importer.get_modules("myvar") + ) def test_removing_files(self): self.mod1.write("myvar = None\n") From f57fdbbb1199eec4a98aa8c6ce1d44efba3c2049 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 17:58:17 -0500 Subject: [PATCH 11/62] migrate to pytest --- ropetest/contrib/autoimporttest.py | 295 +++++++++++++++-------------- 1 file changed, 150 insertions(+), 145 deletions(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index e4d8f3071..493739d13 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -1,161 +1,166 @@ -try: - import unittest2 as unittest -except ImportError: - import unittest +import pytest from rope.contrib import autoimport from ropetest import testutils -class AutoImportTest(unittest.TestCase): - def setUp(self): - super(AutoImportTest, self).setUp() - self.project = testutils.sample_project(extension_modules=["sys"]) - self.project_name = "sample_project" - self.mod1 = testutils.create_module(self.project, "mod1") - self.pkg = testutils.create_package(self.project, "pkg") - self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=False, memory=True) - - def tearDown(self): - testutils.remove_project(self.project) - self.importer.close() - super(AutoImportTest, self).tearDown() - - def test_simple_case(self): - self.assertEqual([], self.importer.import_assist("A")) - - def test_update_resource(self): - self.mod1.write("myvar = None\n") - self.importer.update_resource(self.mod1) - self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) - - def test_update_module(self): - self.mod1.write("myvar = None") - self.importer.update_module("mod1") - self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) - - def test_update_non_existent_module(self): - self.importer.update_module("does_not_exists_this") - self.assertEqual([], self.importer.import_assist("myva")) - - def test_module_with_syntax_errors(self): - self.mod1.write("this is a syntax error\n") - self.importer.update_resource(self.mod1) - self.assertEqual([], self.importer.import_assist("myva")) - - def test_excluding_imported_names(self): - self.mod1.write("import pkg\n") - self.importer.update_resource(self.mod1) - self.assertEqual([], self.importer.import_assist("pkg")) - - def test_get_modules(self): - self.mod1.write("myvar = None\n") - self.importer.update_resource(self.mod1) - self.assertEqual( - [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") - ) +@pytest.fixture +def project(): + project = testutils.sample_project(extension_modules=["sys"]) + yield project + testutils.remove_project(project) - def test_get_modules_inside_packages(self): - self.mod1.write("myvar = None\n") - self.mod2.write("myvar = None\n") - self.importer.update_resource(self.mod1) - self.importer.update_resource(self.mod2) - self.assertEqual( - set([f"{self.project_name}.mod1", f"{self.project_name}.pkg.mod2"]), - set(self.importer.get_modules("myvar")), - ) - def test_trivial_insertion_line(self): - result = self.importer.find_insertion_line("") - self.assertEqual(1, result) +@pytest.fixture +def project_name(): + return "sample_project" - def test_insertion_line(self): - result = self.importer.find_insertion_line("import mod\n") - self.assertEqual(2, result) - def test_insertion_line_with_pydocs(self): - result = self.importer.find_insertion_line('"""docs\n\ndocs"""\nimport mod\n') - self.assertEqual(5, result) +@pytest.fixture +def importer(project): + importer = autoimport.AutoImport(project, observe=False, memory=True) + yield importer + importer.close() - def test_insertion_line_with_multiple_imports(self): - result = self.importer.find_insertion_line("import mod1\n\nimport mod2\n") - self.assertEqual(4, result) - def test_insertion_line_with_blank_lines(self): - result = self.importer.find_insertion_line("import mod1\n\n# comment\n") - self.assertEqual(2, result) +@pytest.fixture +def mod1(importer, project): + mod1 = testutils.create_module(project, "mod1") + yield mod1 - def test_empty_cache(self): - self.mod1.write("myvar = None\n") - self.importer.update_resource(self.mod1) - self.assertEqual( - [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") - ) - self.importer.clear_cache() - self.assertEqual([], self.importer.get_modules("myvar")) - - def test_not_caching_underlined_names(self): - self.mod1.write("_myvar = None\n") - self.importer.update_resource(self.mod1, underlined=False) - self.assertEqual([], self.importer.get_modules("_myvar")) - self.importer.update_resource(self.mod1, underlined=True) - self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) - - def test_caching_underlined_names_passing_to_the_constructor(self): - importer = autoimport.AutoImport(self.project, False, True) - self.mod1.write("_myvar = None\n") - importer.update_resource(self.mod1) - self.assertEqual(["mod1"], importer.get_modules("_myvar")) - - def test_name_locations(self): - self.mod1.write("myvar = None\n") - self.importer.update_resource(self.mod1) - self.assertEqual([(self.mod1, 1)], self.importer.get_name_locations("myvar")) - - def test_name_locations_with_multiple_occurrences(self): - self.mod1.write("myvar = None\n") - self.mod2.write("\nmyvar = None\n") - self.importer.update_resource(self.mod1) - self.importer.update_resource(self.mod2) - self.assertEqual( - set([(self.mod1, 1), (self.mod2, 2)]), - set(self.importer.get_name_locations("myvar")), - ) - def test_handling_builtin_modules(self): - self.importer.update_module("sys") - self.assertTrue("sys" in self.importer.get_modules("exit")) - - -class AutoImportObservingTest(unittest.TestCase): - def setUp(self): - super(AutoImportObservingTest, self).setUp() - self.project = testutils.sample_project() - self.mod1 = testutils.create_module(self.project, "mod1") - self.pkg = testutils.create_package(self.project, "pkg") - self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=True, memory=True) - self.project_name = "sample_project" - def tearDown(self): - testutils.remove_project(self.project) - self.importer.close() - super(AutoImportObservingTest, self).tearDown() - - def test_writing_files(self): - self.mod1.write("myvar = None\n") - self.assertEqual( - [f"{self.project_name}.mod1"], self.importer.get_modules("myvar") - ) +@pytest.fixture +def pkg(importer, project): + pkg = testutils.create_package(project, "pkg") + yield pkg + + +@pytest.fixture +def importer_observing(project): + project = testutils.sample_project() + importer = autoimport.AutoImport(project, observe=True, memory=True) + yield importer + importer.close() + + +@pytest.fixture +def mod2(importer, project, pkg): + mod2 = testutils.create_module(project, "mod2") + yield mod2 + + +class TestAutoImport: + def test_simple_case(self, importer): + assert [] == importer.import_assist("A") + + def test_update_resource(self, importer, mod1): + mod1.write("myvar = None\n") + importer.update_resource(mod1) + assert [("myvar", "mod1")] == importer.import_assist("myva") + + def test_update_module(self, importer, mod1): + mod1.write("myvar = None") + importer.update_module("mod1") + assert [("myvar", "mod1")] == importer.import_assist("myva") + + def test_update_non_existent_module(self, importer): + importer.update_module("does_not_exists_this") + assert [] == importer.import_assist("myva") + + def test_module_with_syntax_errors(self, importer, mod1): + mod1.write("this is a syntax error\n") + importer.update_resource(mod1) + assert [] == importer.import_assist("myva") + + def test_excluding_imported_names(self, mod1, importer): + mod1.write("import pkg\n") + importer.update_resource(mod1) + assert [] == importer.import_assist("pkg") + + def test_get_modules(self, mod1, importer, project_name): + mod1.write("myvar = None\n") + importer.update_resource(mod1) + assert [f"{project_name}.mod1"] == importer.get_modules("myvar") - def test_moving_files(self): - self.mod1.write("myvar = None\n") - self.mod1.move("mod3.py") - self.assertEqual( - [f"{self.project_name}.mod3"], self.importer.get_modules("myvar") + def test_get_modules_inside_packages(self, mod1, mod2, importer, project_name): + mod1.write("myvar = None\n") + mod2.write("myvar = None\n") + importer.update_resource(mod1) + importer.update_resource(mod2) + assert set([f"{project_name}.mod1", f"{project_name}.pkg.mod2"]) == set( + importer.get_modules("myvar") ) - def test_removing_files(self): - self.mod1.write("myvar = None\n") - self.mod1.remove() - self.assertEqual([], self.importer.get_modules("myvar")) + def test_trivial_insertion_line(self, importer): + result = importer.find_insertion_line("") + assert 1 == result + + def test_insertion_line(self, importer): + result = importer.find_insertion_line("import mod\n") + assert 2 == result + + def test_insertion_line_with_pydocs(self, importer): + result = importer.find_insertion_line('"""docs\n\ndocs"""\nimport mod\n') + assert 5 == result + + def test_insertion_line_with_multiple_imports(self, importer): + result = importer.find_insertion_line("import mod1\n\nimport mod2\n") + assert 4 == result + + def test_insertion_line_with_blank_lines(self, importer): + result = importer.find_insertion_line("import mod1\n\n# comment\n") + assert 2 == result + + def test_empty_cache(self, importer, mod1, project_name): + mod1.write("myvar = None\n") + importer.update_resource(mod1) + assert [f"{project_name}.mod1"] == importer.get_modules("myvar") + + importer.clear_cache() + assert [] == importer.get_modules("myvar") + + def test_not_caching_underlined_names(self, importer, mod1): + mod1.write("_myvar = None\n") + importer.update_resource(mod1, underlined=False) + assert [] == importer.get_modules("_myvar") + importer.update_resource(mod1, underlined=True) + assert ["mod1"] == importer.get_modules("_myvar") + + def test_caching_underlined_names_passing_to_the_constructor(self, project, mod1): + importer = autoimport.AutoImport(project, False, True) + mod1.write("_myvar = None\n") + importer.update_resource(mod1) + assert ["mod1"] == importer.get_modules("_myvar") + importer.close() + + def test_name_locations(self, importer, mod1): + mod1.write("myvar = None\n") + importer.update_resource(mod1) + assert [(mod1, 1)] == importer.get_name_locations("myvar") + + def test_name_locations_with_multiple_occurrences(self, mod1, mod2, importer): + mod1.write("myvar = None\n") + mod2.write("\nmyvar = None\n") + importer.update_resource(mod1) + importer.update_resource(mod2) + assert set([(mod1, 1), (mod2, 2)]) == set(importer.get_name_locations("myvar")) + + def test_handling_builtin_modules(self, importer): + importer.update_module("sys") + assert "sys" == importer.get_modules("exit") + + +class TestAutoImportObserving: + def test_writing_files(self, importer_observing, mod1, project_name): + mod1.write("myvar = None\n") + assert [f"{project_name}.mod1"] == importer_observing.get_modules("myvar") + + def test_moving_files(self, importer_observing, mod1, project_name): + mod1.write("myvar = None\n") + mod1.move("mod3.py") + assert [f"{project_name}.mod3"] == importer_observing.get_modules("myvar") + + def test_removing_files(self, importer_observing, mod1): + mod1.write("myvar = None\n") + mod1.remove() + assert [] == importer_observing.get_modules("myvar") From b49dda1f0dad9072b4715812182e87904de24245 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 18:38:50 -0500 Subject: [PATCH 12/62] handle observer module names correctly --- rope/contrib/autoimport.py | 34 ++++++++++++++++++++---------- ropetest/contrib/autoimporttest.py | 22 +++++++++---------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index e1ebd3190..de1c014ef 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -213,7 +213,9 @@ def _sort_and_deduplicate_tuple( if len(results) == 0: return [] results.sort(key=lambda y: y[-1]) - results_sorted = list(zip(*results))[:-1] + results_sorted = [] + for result in results: + results_sorted.append(result[:-1]) return list(OrderedDict.fromkeys(results_sorted)) @@ -241,17 +243,19 @@ def __init__(self, project, observe=True, underlined=False, memory=True): self.underlined = underlined db_path = ":memory:" if memory else f"{project.ropefolder.path}/autoimport.db" self.connection = sqlite3.connect(db_path) - self.connection.execute( - "create table if not exists names(name TEXT, module TEXT, package TEXT, source INTEGER)" - ) + self._setup_db() self._check_all() - # XXX: using a filtered observer - observer = resourceobserver.ResourceObserver( - changed=self._changed, moved=self._moved, removed=self._removed - ) if observe: + observer = resourceobserver.ResourceObserver( + changed=self._changed, moved=self._moved, removed=self._removed + ) project.add_observer(observer) + def _setup_db(self): + self.connection.execute( + "create table if not exists names(name TEXT, module TEXT, package TEXT, source INTEGER)" + ) + def import_assist(self, starting): """Return a list of ``(name, module)`` tuples @@ -387,6 +391,7 @@ def clear_cache(self): """ self.connection.execute("drop table names") + self._setup_db() self.connection.commit() def find_insertion_line(self, code): @@ -432,9 +437,10 @@ def _changed(self, resource): if not resource.is_folder(): self.update_resource(resource) - def _moved(self, resource, newresource): + def _moved(self, resource: Resource, newresource: Resource): if not resource.is_folder(): - modname = libutils.modname(resource) + modname = self._modname(resource) + print(modname) self._del_if_exist(modname) self.update_resource(newresource) @@ -442,9 +448,15 @@ def _del_if_exist(self, module_name): self.connection.execute("delete from names where module = ?", (module_name,)) self.connection.commit() + def _modname(self, resource: Resource): + resource_path: pathlib.Path = pathlib.Path(resource.real_path) + package_path: pathlib.Path = pathlib.Path(self.project.address) + resource_modname: str = _get_modname_from_path(resource_path, package_path) + return resource_modname + def _removed(self, resource): if not resource.is_folder(): - modname = libutils.modname(resource) + modname = self._modname(resource) self._del_if_exist(modname) def _add_names(self, names: List[Name]): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 493739d13..45c2f7966 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -37,7 +37,6 @@ def pkg(importer, project): @pytest.fixture def importer_observing(project): - project = testutils.sample_project() importer = autoimport.AutoImport(project, observe=True, memory=True) yield importer importer.close() @@ -45,7 +44,7 @@ def importer_observing(project): @pytest.fixture def mod2(importer, project, pkg): - mod2 = testutils.create_module(project, "mod2") + mod2 = testutils.create_module(project, "mod2", pkg) yield mod2 @@ -53,15 +52,15 @@ class TestAutoImport: def test_simple_case(self, importer): assert [] == importer.import_assist("A") - def test_update_resource(self, importer, mod1): + def test_update_resource(self, importer, mod1, project_name): mod1.write("myvar = None\n") importer.update_resource(mod1) - assert [("myvar", "mod1")] == importer.import_assist("myva") + assert [("myvar", f"{project_name}.mod1")] == importer.import_assist("myva") - def test_update_module(self, importer, mod1): + def test_update_module(self, importer, mod1, project_name): mod1.write("myvar = None") - importer.update_module("mod1") - assert [("myvar", "mod1")] == importer.import_assist("myva") + importer.update_module(f"{project_name}.mod1") + assert [("myvar", f"{project_name}.mod1")] == importer.import_assist("myva") def test_update_non_existent_module(self, importer): importer.update_module("does_not_exists_this") @@ -87,9 +86,10 @@ def test_get_modules_inside_packages(self, mod1, mod2, importer, project_name): mod2.write("myvar = None\n") importer.update_resource(mod1) importer.update_resource(mod2) - assert set([f"{project_name}.mod1", f"{project_name}.pkg.mod2"]) == set( - importer.get_modules("myvar") - ) + assert [ + f"{project_name}.mod1", + f"{project_name}.pkg.mod2", + ] == importer.get_modules("myvar") def test_trivial_insertion_line(self, importer): result = importer.find_insertion_line("") @@ -143,7 +143,7 @@ def test_name_locations_with_multiple_occurrences(self, mod1, mod2, importer): mod2.write("\nmyvar = None\n") importer.update_resource(mod1) importer.update_resource(mod2) - assert set([(mod1, 1), (mod2, 2)]) == set(importer.get_name_locations("myvar")) + assert [(mod1, 1), (mod2, 2)] == importer.get_name_locations("myvar") def test_handling_builtin_modules(self, importer): importer.update_module("sys") From 3f932579d9a338a80d784f501f16b48fae1efb27 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 19:52:23 -0500 Subject: [PATCH 13/62] error propogation --- rope/contrib/autoimport.py | 8 ++------ ropetest/contrib/autoimporttest.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index de1c014ef..b5ddb2715 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -263,7 +263,7 @@ def import_assist(self, starting): that starts with `starting`. """ results = self.connection.execute( - "select name, module, source from names where name like (?)", + "select name, module, source from names WHERE name LIKE (?)", (starting + "%",), ).fetchall() for result in results: @@ -374,7 +374,6 @@ def generate_modules_cache( with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(_find_all_names_in_package, packages): self._add_names(name_list) - self.connection.commit() def update_module(self, module: str): self.generate_modules_cache([module]) @@ -422,15 +421,13 @@ def update_resource(self, resource: Resource, underlined: bool = False): if package_tuple is None: return None package_name = package_tuple[0] - print((resource_path, resource_modname, package_path.name, Source.PROJECT)) names = _get_names_from_file( resource_path, resource_modname, package_name, Source.PROJECT, - underlined, + underlined=underlined, ) - print(names) self._add_names(names) def _changed(self, resource): @@ -440,7 +437,6 @@ def _changed(self, resource): def _moved(self, resource: Resource, newresource: Resource): if not resource.is_folder(): modname = self._modname(resource) - print(modname) self._del_if_exist(modname) self.update_resource(newresource) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 45c2f7966..e3fdf77ff 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -7,6 +7,7 @@ @pytest.fixture def project(): project = testutils.sample_project(extension_modules=["sys"]) + project.set("python_path", [project.address]) yield project testutils.remove_project(project) @@ -59,8 +60,8 @@ def test_update_resource(self, importer, mod1, project_name): def test_update_module(self, importer, mod1, project_name): mod1.write("myvar = None") - importer.update_module(f"{project_name}.mod1") - assert [("myvar", f"{project_name}.mod1")] == importer.import_assist("myva") + importer.update_module("mod1") + assert [("myvar", "mod1")] == importer.import_assist("myva") def test_update_non_existent_module(self, importer): importer.update_module("does_not_exists_this") @@ -119,18 +120,20 @@ def test_empty_cache(self, importer, mod1, project_name): importer.clear_cache() assert [] == importer.get_modules("myvar") - def test_not_caching_underlined_names(self, importer, mod1): + def test_not_caching_underlined_names(self, importer, mod1, project_name): mod1.write("_myvar = None\n") importer.update_resource(mod1, underlined=False) assert [] == importer.get_modules("_myvar") importer.update_resource(mod1, underlined=True) - assert ["mod1"] == importer.get_modules("_myvar") + assert [f"{project_name}.mod1"] == importer.get_modules("_myvar") - def test_caching_underlined_names_passing_to_the_constructor(self, project, mod1): + def test_caching_underlined_names_passing_to_the_constructor( + self, project, mod1, project_name + ): importer = autoimport.AutoImport(project, False, True) mod1.write("_myvar = None\n") importer.update_resource(mod1) - assert ["mod1"] == importer.get_modules("_myvar") + assert [f"{project_name}.mod1"] == importer.get_modules("_myvar") importer.close() def test_name_locations(self, importer, mod1): From 929052873823b45f20f991f62b298c9674d26ae3 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 20:04:29 -0500 Subject: [PATCH 14/62] get name locations --- rope/contrib/autoimport.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index b5ddb2715..9f81e298b 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -368,6 +368,7 @@ def generate_modules_cache( else: for modname in modules: package_path = self._find_package_path(modname) + print(package_path) if package_path is None: continue packages.append(package_path) @@ -382,6 +383,27 @@ def close(self): self.connection.commit() self.connection.close() + def get_name_locations(self, name): + """Return a list of ``(resource, lineno)`` tuples""" + result = [] + names = self.connection.execute( + "select module from names where name like (?)", (name,) + ).fetchall() + for module in names: + try: + module_name = module[0].removeprefix(f"{self._project_name}.") + pymodule = self.project.get_module(module_name) + if name in pymodule: + pyname = pymodule[name] + module, lineno = pyname.get_definition_location() + if module is not None: + resource = module.get_module().get_resource() + if resource is not None and lineno is not None: + result.append((resource, lineno)) + except exceptions.ModuleNotFoundError: + pass + return result + def clear_cache(self): """Clear all entries in global-name cache @@ -444,6 +466,14 @@ def _del_if_exist(self, module_name): self.connection.execute("delete from names where module = ?", (module_name,)) self.connection.commit() + @property + def _project_name(self): + package_path: pathlib.Path = pathlib.Path(self.project.address) + package_tuple = _get_package_name_from_path(package_path) + if package_tuple is None: + return None + return package_tuple[0] + def _modname(self, resource: Resource): resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) From b37fad52a5aa2fed6213c19bbde30e0092f931df Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 21:19:33 -0500 Subject: [PATCH 15/62] update generate_module_cache to exclude the project --- rope/contrib/autoimport.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index 9f81e298b..f726a270f 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -342,7 +342,9 @@ def generate_modules_cache( modules=None, task_handle=taskhandle.NullTaskHandle(), ): - """Generate global name cache for modules listed in `modules`""" + """Generate global name cache for external modules listed in `modules`. + If no modules are provided, it will generate a cache for every module avalible. + This method searches in your sys.path and configured python folders""" job_set = task_handle.create_jobset( "Generating autoimport cache for modules", "all" if modules is None else len(modules), @@ -372,6 +374,10 @@ def generate_modules_cache( if package_path is None: continue packages.append(package_path) + try: + packages.remove(self._project_name) + except ValueError: + pass with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(_find_all_names_in_package, packages): self._add_names(name_list) From 65f0c725bd6d70a368d1efe21ccd096f558d5bb3 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 21:53:23 -0500 Subject: [PATCH 16/62] add builtin handling --- rope/contrib/autoimport.py | 63 +++++++++++++++++++++++------- ropetest/contrib/autoimporttest.py | 2 +- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport.py index f726a270f..141836167 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport.py @@ -1,4 +1,5 @@ import ast +import inspect import pathlib import re import sqlite3 @@ -6,7 +7,8 @@ from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from enum import Enum -from typing import List, Optional, Set, Tuple +from importlib import import_module +from typing import Iterable, List, Optional, Set, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project @@ -17,9 +19,10 @@ class Source(Enum): PROJECT = 0 # Obviously any project packages come first MANUAL = 1 # Any packages manually added are probably important to the user - STANDARD = 2 # We want to favor standard library items - SITE_PACKAGE = 3 - UNKNOWN = 4 + BUILTIN = 2 + STANDARD = 3 # We want to favor standard library items + SITE_PACKAGE = 4 + UNKNOWN = 5 Name = Tuple[str, str, str, int] @@ -73,6 +76,29 @@ class PackageType(Enum): SINGLE_FILE = 3 # a .py file +def _get_names_from_builtins( + underlined: bool = False, packages: Iterable[str] = sys.builtin_module_names +) -> List[Name]: + """Gets names from builtin modules. These are the only modules it is safe to get the names from""" + results: List[Name] = [] + for package in packages: + if package == "builtins": + continue # Builtins is redundant since you don't have to import it. + if underlined or not package.startswith("_"): + module = import_module(package) + if hasattr(module, "__all__"): + for name in module.__all__: + results.append((str(name), package, package, Source.BUILTIN.value)) + for name, value in inspect.getmembers(module): + if underlined or not name.startswith("_"): + if inspect.isclass(value) or inspect.isfunction(value) or inspect.isbuiltin(value): + results.append( + (str(name), package, package, Source.BUILTIN.value) + ) + + return results + + def _get_package_name_from_path( package_path: pathlib.Path, ) -> Optional[Tuple[str, PackageType]]: @@ -316,7 +342,7 @@ def get_all(self) -> List[Name]: def generate_resource_cache( self, - resources=None, + resources: List[Resource] = None, underlined: bool = False, task_handle=taskhandle.NullTaskHandle(), ): @@ -325,7 +351,6 @@ def generate_resource_cache( If `resources` is a list of `rope.base.resource.File`, only those files are searched; otherwise all python modules in the project are cached. - """ if resources is None: resources = self.project.get_python_files() @@ -339,18 +364,21 @@ def generate_resource_cache( def generate_modules_cache( self, - modules=None, + modules: List[str] = None, task_handle=taskhandle.NullTaskHandle(), ): """Generate global name cache for external modules listed in `modules`. If no modules are provided, it will generate a cache for every module avalible. - This method searches in your sys.path and configured python folders""" + This method searches in your sys.path and configured python folders. + Do not use this for generating your own project's internal names, use generate_resource_cache for that instead.""" job_set = task_handle.create_jobset( "Generating autoimport cache for modules", "all" if modules is None else len(modules), ) packages: List[pathlib.Path] = [] if modules is None: + # Get builtins first + self._add_names(_get_names_from_builtins(self.underlined)) folders = self.project.get_python_path_folders() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): @@ -369,13 +397,20 @@ def generate_modules_cache( else: for modname in modules: - package_path = self._find_package_path(modname) - print(package_path) - if package_path is None: - continue - packages.append(package_path) + if modname in sys.builtin_module_names: + names = _get_names_from_builtins( + underlined=True, packages=[modname] + ) + self._add_names(names) + else: + package_path = self._find_package_path(modname) + if package_path is None: + continue + packages.append(package_path) try: - packages.remove(self._project_name) + packages.remove( + self._project_name + ) # Don't want to generate the cache for the user's project except ValueError: pass with ProcessPoolExecutor() as exectuor: diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index e3fdf77ff..0ad197da0 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -150,7 +150,7 @@ def test_name_locations_with_multiple_occurrences(self, mod1, mod2, importer): def test_handling_builtin_modules(self, importer): importer.update_module("sys") - assert "sys" == importer.get_modules("exit") + assert ["sys"] == importer.get_modules("exit") class TestAutoImportObserving: From 08e876872fcf335853ab669b75b807e0eb8cd281 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 22:25:08 -0500 Subject: [PATCH 17/62] split into multiple files --- rope/contrib/autoimport/__init__.py | 2 + rope/contrib/{ => autoimport}/autoimport.py | 251 +------------------- rope/contrib/autoimport/defs.py | 22 ++ rope/contrib/autoimport/parse.py | 165 +++++++++++++ rope/contrib/autoimport/utils.py | 82 +++++++ 5 files changed, 277 insertions(+), 245 deletions(-) create mode 100644 rope/contrib/autoimport/__init__.py rename rope/contrib/{ => autoimport}/autoimport.py (59%) create mode 100644 rope/contrib/autoimport/defs.py create mode 100644 rope/contrib/autoimport/parse.py create mode 100644 rope/contrib/autoimport/utils.py diff --git a/rope/contrib/autoimport/__init__.py b/rope/contrib/autoimport/__init__.py new file mode 100644 index 000000000..d185a7943 --- /dev/null +++ b/rope/contrib/autoimport/__init__.py @@ -0,0 +1,2 @@ +from .autoimport import AutoImport +__all__ = ["AutoImport"] diff --git a/rope/contrib/autoimport.py b/rope/contrib/autoimport/autoimport.py similarity index 59% rename from rope/contrib/autoimport.py rename to rope/contrib/autoimport/autoimport.py index 141836167..c29de8ac4 100644 --- a/rope/contrib/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -1,248 +1,20 @@ -import ast -import inspect import pathlib import re import sqlite3 import sys -from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor -from enum import Enum -from importlib import import_module -from typing import Iterable, List, Optional, Set, Tuple +from typing import List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource from rope.refactor import importutils - -class Source(Enum): - PROJECT = 0 # Obviously any project packages come first - MANUAL = 1 # Any packages manually added are probably important to the user - BUILTIN = 2 - STANDARD = 3 # We want to favor standard library items - SITE_PACKAGE = 4 - UNKNOWN = 5 - - -Name = Tuple[str, str, str, int] - - -def _get_names( - modpath: pathlib.Path, - modname: str, - package_name: str, - package_source: Source, - underlined: bool = False, -) -> List[Name]: - """Update the cache for global names in `modname` module - - `modname` is the name of a module. - """ - # TODO use __all__ parsing if avalible - if modpath.is_dir(): - names: List[Name] - if modpath / "__init__.py": - names = _get_names_from_file( - modpath / "__init__.py", - modname, - package_name, - package_source, - only_all=True, - ) - if len(names) > 0: - return names - names = [] - for file in modpath.glob("*.py"): - names.extend( - _get_names_from_file( - file, - modname + f".{file.name.removesuffix('.py')}", - package_name, - package_source, - underlined=underlined, - ) - ) - return names - else: - return _get_names_from_file( - modpath, modname, package_name, package_source, underlined=underlined - ) - - -class PackageType(Enum): - STANDARD = 1 # Just a folder - COMPILED = 2 # .so module - SINGLE_FILE = 3 # a .py file - - -def _get_names_from_builtins( - underlined: bool = False, packages: Iterable[str] = sys.builtin_module_names -) -> List[Name]: - """Gets names from builtin modules. These are the only modules it is safe to get the names from""" - results: List[Name] = [] - for package in packages: - if package == "builtins": - continue # Builtins is redundant since you don't have to import it. - if underlined or not package.startswith("_"): - module = import_module(package) - if hasattr(module, "__all__"): - for name in module.__all__: - results.append((str(name), package, package, Source.BUILTIN.value)) - for name, value in inspect.getmembers(module): - if underlined or not name.startswith("_"): - if inspect.isclass(value) or inspect.isfunction(value) or inspect.isbuiltin(value): - results.append( - (str(name), package, package, Source.BUILTIN.value) - ) - - return results - - -def _get_package_name_from_path( - package_path: pathlib.Path, -) -> Optional[Tuple[str, PackageType]]: - package_name = package_path.name - if package_name.endswith(".egg-info"): - return None - if package_name.endswith(".so"): - name = package_name.split(".")[0] - return (name, PackageType.COMPILED) - # TODO add so handling - if package_name.endswith(".py"): - stripped_name = package_name.removesuffix(".py") - return (stripped_name, PackageType.SINGLE_FILE) - return (package_name, PackageType.STANDARD) - - -def _get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: - package_name: str = package_path.name - modname = ( - modpath.relative_to(package_path) - .as_posix() - .removesuffix(".py") - .replace("/", ".") - ) - modname = package_name if modname == "." else package_name + "." + modname - return modname - - -def _find_all_names_in_package( - package_path: pathlib.Path, - recursive=True, - package_source: Source = None, - underlined: bool = False, -) -> List[Name]: - package_tuple = _get_package_name_from_path(package_path) - if package_tuple is None: - return [] - package_name, package_type = package_tuple - if package_source is None: - package_source = get_package_source(package_path) - modules: List[Tuple[pathlib.Path, str]] = [] - if package_type is PackageType.SINGLE_FILE: - modules.append((package_path, package_name)) - elif package_type is PackageType.COMPILED: - return [] - elif recursive: - for sub in _submodules(package_path): - modname = _get_modname_from_path(sub, package_path) - if underlined or modname.__contains__("_"): - continue # Exclude private items - modules.append((sub, modname)) - else: - modules.append((package_path, package_name)) - result: List[Name] = [] - for module in modules: - result.extend( - _get_names(module[0], module[1], package_name, package_source, underlined) - ) - return result - - -def _get_names_from_file( - module: pathlib.Path, - modname: str, - package: str, - package_source: Source, - only_all: bool = False, - underlined: bool = False, -) -> List[Name]: - with open(module, mode="rb") as file: - try: - root_node = ast.parse(file.read()) - except SyntaxError as e: - print(e) - return [] - results: List[Name] = [] - for node in ast.iter_child_nodes(root_node): - node_names: List[str] = [] - if isinstance(node, ast.Assign): - for target in node.targets: - try: - assert isinstance(target, ast.Name) - if target.id == "__all__": - # TODO add tuple handling - all_results: List[Name] = [] - assert isinstance(node.value, ast.List) - for item in node.value.elts: - assert isinstance(item, ast.Constant) - all_results.append( - ( - str(item.value), - modname, - package, - package_source.value, - ) - ) - return all_results - else: - node_names.append(target.id) - except (AttributeError, AssertionError): - # TODO handle tuple assignment - pass - elif isinstance(node, (ast.FunctionDef, ast.ClassDef)): - node_names = [node.name] - for node_name in node_names: - if underlined or not node_name.startswith("_"): - results.append((node_name, modname, package, package_source.value)) - if only_all: - return [] - return results - - -def get_package_source( - package: pathlib.Path, project: Optional[Project] = None -) -> Source: - """Detect the source of a given package. Rudimentary implementation.""" - if project is not None and package.as_posix().__contains__(project.address): - return Source.PROJECT - if package.as_posix().__contains__("site-packages"): - return Source.SITE_PACKAGE - if package.as_posix().startswith(sys.prefix): - return Source.STANDARD - else: - return Source.UNKNOWN - - -def _sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: - if len(results) == 0: - return [] - results.sort(key=lambda y: y[-1]) - results_sorted = list(zip(*results))[0] - return list(OrderedDict.fromkeys(results_sorted)) - - -def _sort_and_deduplicate_tuple( - results: List[Tuple[str, str, int]] -) -> List[Tuple[str, str]]: - if len(results) == 0: - return [] - results.sort(key=lambda y: y[-1]) - results_sorted = [] - for result in results: - results_sorted.append(result[:-1]) - return list(OrderedDict.fromkeys(results_sorted)) +from .defs import Name, Source +from .parse import (_find_all_names_in_package, _get_names_from_builtins, + _get_names_from_file) +from .utils import (_get_modname_from_path, _get_package_name_from_path, + _sort_and_deduplicate, _sort_and_deduplicate_tuple) class AutoImport(object): @@ -555,14 +327,3 @@ def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: if package_tuple[0] == package_name: return package return None - - -def _submodules(mod: pathlib.Path) -> Set[pathlib.Path]: - """Simple submodule finder that doesn't try to import anything""" - result = set() - if mod.is_dir() and (mod / "__init__.py").exists(): - result.add(mod) - for child in mod.iterdir(): - result |= _submodules(child) - return result - return result diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py new file mode 100644 index 000000000..63e6fc1b6 --- /dev/null +++ b/rope/contrib/autoimport/defs.py @@ -0,0 +1,22 @@ +"""Definitions of types for the Autoimport program""" +from typing import Tuple + +from enum import Enum + + +class Source(Enum): + PROJECT = 0 # Obviously any project packages come first + MANUAL = 1 # Any packages manually added are probably important to the user + BUILTIN = 2 + STANDARD = 3 # We want to favor standard library items + SITE_PACKAGE = 4 + UNKNOWN = 5 + + +Name = Tuple[str, str, str, int] + + +class PackageType(Enum): + STANDARD = 1 # Just a folder + COMPILED = 2 # .so module + SINGLE_FILE = 3 # a .py file diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py new file mode 100644 index 000000000..1ca4fbcca --- /dev/null +++ b/rope/contrib/autoimport/parse.py @@ -0,0 +1,165 @@ +"""Files to parse the source code of a python file, so object, or builtin module for autoimport names""" + +import ast +import inspect +import pathlib +import sys +from importlib import import_module +from typing import Iterable, List, Tuple + +from .defs import Name, PackageType, Source +from .utils import (_get_modname_from_path, _get_package_name_from_path, + _submodules, get_package_source) + + +def _get_names( + modpath: pathlib.Path, + modname: str, + package_name: str, + package_source: Source, + underlined: bool = False, +) -> List[Name]: + """Update the cache for global names in `modname` module + + `modname` is the name of a module. + """ + # TODO use __all__ parsing if avalible + if modpath.is_dir(): + names: List[Name] + if modpath / "__init__.py": + names = _get_names_from_file( + modpath / "__init__.py", + modname, + package_name, + package_source, + only_all=True, + ) + if len(names) > 0: + return names + names = [] + for file in modpath.glob("*.py"): + names.extend( + _get_names_from_file( + file, + modname + f".{file.name.removesuffix('.py')}", + package_name, + package_source, + underlined=underlined, + ) + ) + return names + else: + return _get_names_from_file( + modpath, modname, package_name, package_source, underlined=underlined + ) + + +def _get_names_from_file( + module: pathlib.Path, + modname: str, + package: str, + package_source: Source, + only_all: bool = False, + underlined: bool = False, +) -> List[Name]: + with open(module, mode="rb") as file: + try: + root_node = ast.parse(file.read()) + except SyntaxError as e: + print(e) + return [] + results: List[Name] = [] + for node in ast.iter_child_nodes(root_node): + node_names: List[str] = [] + if isinstance(node, ast.Assign): + for target in node.targets: + try: + assert isinstance(target, ast.Name) + if target.id == "__all__": + # TODO add tuple handling + all_results: List[Name] = [] + assert isinstance(node.value, ast.List) + for item in node.value.elts: + assert isinstance(item, ast.Constant) + all_results.append( + ( + str(item.value), + modname, + package, + package_source.value, + ) + ) + return all_results + else: + node_names.append(target.id) + except (AttributeError, AssertionError): + # TODO handle tuple assignment + pass + elif isinstance(node, (ast.FunctionDef, ast.ClassDef)): + node_names = [node.name] + for node_name in node_names: + if underlined or not node_name.startswith("_"): + results.append((node_name, modname, package, package_source.value)) + if only_all: + return [] + return results + + +def _find_all_names_in_package( + package_path: pathlib.Path, + recursive=True, + package_source: Source = None, + underlined: bool = False, +) -> List[Name]: + package_tuple = _get_package_name_from_path(package_path) + if package_tuple is None: + return [] + package_name, package_type = package_tuple + if package_source is None: + package_source = get_package_source(package_path) + modules: List[Tuple[pathlib.Path, str]] = [] + if package_type is PackageType.SINGLE_FILE: + modules.append((package_path, package_name)) + elif package_type is PackageType.COMPILED: + return [] + elif recursive: + for sub in _submodules(package_path): + modname = _get_modname_from_path(sub, package_path) + if underlined or modname.__contains__("_"): + continue # Exclude private items + modules.append((sub, modname)) + else: + modules.append((package_path, package_name)) + result: List[Name] = [] + for module in modules: + result.extend( + _get_names(module[0], module[1], package_name, package_source, underlined) + ) + return result + + +def _get_names_from_builtins( + underlined: bool = False, packages: Iterable[str] = sys.builtin_module_names +) -> List[Name]: + """Gets names from builtin modules. These are the only modules it is safe to get the names from""" + results: List[Name] = [] + for package in packages: + if package == "builtins": + continue # Builtins is redundant since you don't have to import it. + if underlined or not package.startswith("_"): + module = import_module(package) + if hasattr(module, "__all__"): + for name in module.__all__: + results.append((str(name), package, package, Source.BUILTIN.value)) + for name, value in inspect.getmembers(module): + if underlined or not name.startswith("_"): + if ( + inspect.isclass(value) + or inspect.isfunction(value) + or inspect.isbuiltin(value) + ): + results.append( + (str(name), package, package, Source.BUILTIN.value) + ) + + return results diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py new file mode 100644 index 000000000..9d8a49718 --- /dev/null +++ b/rope/contrib/autoimport/utils.py @@ -0,0 +1,82 @@ +"""Utility functions for the autoimport code""" +import pathlib +import sys +from collections import OrderedDict +from typing import List, Optional, Set, Tuple + +from rope.base.project import Project + +from .defs import PackageType, Source + + +def _get_package_name_from_path( + package_path: pathlib.Path, +) -> Optional[Tuple[str, PackageType]]: + package_name = package_path.name + if package_name.endswith(".egg-info"): + return None + if package_name.endswith(".so"): + name = package_name.split(".")[0] + return (name, PackageType.COMPILED) + # TODO add so handling + if package_name.endswith(".py"): + stripped_name = package_name.removesuffix(".py") + return (stripped_name, PackageType.SINGLE_FILE) + return (package_name, PackageType.STANDARD) + + +def _get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: + package_name: str = package_path.name + modname = ( + modpath.relative_to(package_path) + .as_posix() + .removesuffix(".py") + .replace("/", ".") + ) + modname = package_name if modname == "." else package_name + "." + modname + return modname + + +def get_package_source( + package: pathlib.Path, project: Optional[Project] = None +) -> Source: + """Detect the source of a given package. Rudimentary implementation.""" + if project is not None and package.as_posix().__contains__(project.address): + return Source.PROJECT + if package.as_posix().__contains__("site-packages"): + return Source.SITE_PACKAGE + if package.as_posix().startswith(sys.prefix): + return Source.STANDARD + else: + return Source.UNKNOWN + + +def _sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: + if len(results) == 0: + return [] + results.sort(key=lambda y: y[-1]) + results_sorted = list(zip(*results))[0] + return list(OrderedDict.fromkeys(results_sorted)) + + +def _sort_and_deduplicate_tuple( + results: List[Tuple[str, str, int]] +) -> List[Tuple[str, str]]: + if len(results) == 0: + return [] + results.sort(key=lambda y: y[-1]) + results_sorted = [] + for result in results: + results_sorted.append(result[:-1]) + return list(OrderedDict.fromkeys(results_sorted)) + + +def _submodules(mod: pathlib.Path) -> Set[pathlib.Path]: + """Simple submodule finder that doesn't try to import anything""" + result = set() + if mod.is_dir() and (mod / "__init__.py").exists(): + result.add(mod) + for child in mod.iterdir(): + result |= _submodules(child) + return result + return result From c0aba3de5ed2e5854558cf169a365d5aa0902054 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 22:31:24 -0500 Subject: [PATCH 18/62] make private api public within autoimport --- rope/contrib/autoimport/autoimport.py | 36 +++++++++++++-------------- rope/contrib/autoimport/parse.py | 26 +++++++++---------- rope/contrib/autoimport/utils.py | 12 ++++----- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index c29de8ac4..a9a362a82 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -11,10 +11,10 @@ from rope.refactor import importutils from .defs import Name, Source -from .parse import (_find_all_names_in_package, _get_names_from_builtins, - _get_names_from_file) -from .utils import (_get_modname_from_path, _get_package_name_from_path, - _sort_and_deduplicate, _sort_and_deduplicate_tuple) +from .parse import (find_all_names_in_package, get_names_from_builtins, + get_names_from_file) +from .utils import (get_modname_from_path, get_package_name_from_path, + sort_and_deduplicate, sort_and_deduplicate_tuple) class AutoImport(object): @@ -67,7 +67,7 @@ def import_assist(self, starting): for result in results: if not self._check_import(result[1]): del results[result] - return _sort_and_deduplicate_tuple( + return sort_and_deduplicate_tuple( results ) # Remove duplicates from multiple occurences of the same item @@ -88,7 +88,7 @@ def search(self, name) -> List[str]: "Select module, source from names where module LIKE (?)", (name,) ): results.append((f"import {name}", source)) - return _sort_and_deduplicate(results) + return sort_and_deduplicate(results) def get_modules(self, name) -> List[str]: """Return the list of modules that have global `name`""" @@ -98,7 +98,7 @@ def get_modules(self, name) -> List[str]: for result in results: if not self._check_import(result[0]): del results[result] - return _sort_and_deduplicate(results) + return sort_and_deduplicate(results) def get_all_names(self) -> List[str]: """Return the list of all cached global names""" @@ -150,11 +150,11 @@ def generate_modules_cache( packages: List[pathlib.Path] = [] if modules is None: # Get builtins first - self._add_names(_get_names_from_builtins(self.underlined)) + self._add_names(get_names_from_builtins(self.underlined)) folders = self.project.get_python_path_folders() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): - package_tuple = _get_package_name_from_path(package) + package_tuple = get_package_name_from_path(package) if package_tuple is None: continue package_name = package_tuple[0] @@ -170,9 +170,7 @@ def generate_modules_cache( else: for modname in modules: if modname in sys.builtin_module_names: - names = _get_names_from_builtins( - underlined=True, packages=[modname] - ) + names = get_names_from_builtins(underlined=True, packages=[modname]) self._add_names(names) else: package_path = self._find_package_path(modname) @@ -186,7 +184,7 @@ def generate_modules_cache( except ValueError: pass with ProcessPoolExecutor() as exectuor: - for name_list in exectuor.map(_find_all_names_in_package, packages): + for name_list in exectuor.map(find_all_names_in_package, packages): self._add_names(name_list) def update_module(self, module: str): @@ -250,13 +248,13 @@ def update_resource(self, resource: Resource, underlined: bool = False): """Update the cache for global names in `resource`""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = _get_modname_from_path(resource_path, package_path) - package_tuple = _get_package_name_from_path(package_path) + resource_modname: str = get_modname_from_path(resource_path, package_path) + package_tuple = get_package_name_from_path(package_path) underlined = underlined if underlined else self.underlined if package_tuple is None: return None package_name = package_tuple[0] - names = _get_names_from_file( + names = get_names_from_file( resource_path, resource_modname, package_name, @@ -282,7 +280,7 @@ def _del_if_exist(self, module_name): @property def _project_name(self): package_path: pathlib.Path = pathlib.Path(self.project.address) - package_tuple = _get_package_name_from_path(package_path) + package_tuple = get_package_name_from_path(package_path) if package_tuple is None: return None return package_tuple[0] @@ -290,7 +288,7 @@ def _project_name(self): def _modname(self, resource: Resource): resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = _get_modname_from_path(resource_path, package_path) + resource_modname: str = get_modname_from_path(resource_path, package_path) return resource_modname def _removed(self, resource): @@ -321,7 +319,7 @@ def _check_all(self): def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: for folder in self.project.get_python_path_folders(): for package in pathlib.Path(folder.path).iterdir(): - package_tuple = _get_package_name_from_path(package) + package_tuple = get_package_name_from_path(package) if package_tuple is None: continue if package_tuple[0] == package_name: diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 1ca4fbcca..a85c7f184 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -8,11 +8,11 @@ from typing import Iterable, List, Tuple from .defs import Name, PackageType, Source -from .utils import (_get_modname_from_path, _get_package_name_from_path, - _submodules, get_package_source) +from .utils import (get_modname_from_path, get_package_name_from_path, + get_package_source, submodules) -def _get_names( +def get_names( modpath: pathlib.Path, modname: str, package_name: str, @@ -27,7 +27,7 @@ def _get_names( if modpath.is_dir(): names: List[Name] if modpath / "__init__.py": - names = _get_names_from_file( + names = get_names_from_file( modpath / "__init__.py", modname, package_name, @@ -39,7 +39,7 @@ def _get_names( names = [] for file in modpath.glob("*.py"): names.extend( - _get_names_from_file( + get_names_from_file( file, modname + f".{file.name.removesuffix('.py')}", package_name, @@ -49,12 +49,12 @@ def _get_names( ) return names else: - return _get_names_from_file( + return get_names_from_file( modpath, modname, package_name, package_source, underlined=underlined ) -def _get_names_from_file( +def get_names_from_file( module: pathlib.Path, modname: str, package: str, @@ -105,13 +105,13 @@ def _get_names_from_file( return results -def _find_all_names_in_package( +def find_all_names_in_package( package_path: pathlib.Path, recursive=True, package_source: Source = None, underlined: bool = False, ) -> List[Name]: - package_tuple = _get_package_name_from_path(package_path) + package_tuple = get_package_name_from_path(package_path) if package_tuple is None: return [] package_name, package_type = package_tuple @@ -123,8 +123,8 @@ def _find_all_names_in_package( elif package_type is PackageType.COMPILED: return [] elif recursive: - for sub in _submodules(package_path): - modname = _get_modname_from_path(sub, package_path) + for sub in submodules(package_path): + modname = get_modname_from_path(sub, package_path) if underlined or modname.__contains__("_"): continue # Exclude private items modules.append((sub, modname)) @@ -133,12 +133,12 @@ def _find_all_names_in_package( result: List[Name] = [] for module in modules: result.extend( - _get_names(module[0], module[1], package_name, package_source, underlined) + get_names(module[0], module[1], package_name, package_source, underlined) ) return result -def _get_names_from_builtins( +def get_names_from_builtins( underlined: bool = False, packages: Iterable[str] = sys.builtin_module_names ) -> List[Name]: """Gets names from builtin modules. These are the only modules it is safe to get the names from""" diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 9d8a49718..23a7521a6 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -9,7 +9,7 @@ from .defs import PackageType, Source -def _get_package_name_from_path( +def get_package_name_from_path( package_path: pathlib.Path, ) -> Optional[Tuple[str, PackageType]]: package_name = package_path.name @@ -25,7 +25,7 @@ def _get_package_name_from_path( return (package_name, PackageType.STANDARD) -def _get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: +def get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: package_name: str = package_path.name modname = ( modpath.relative_to(package_path) @@ -51,7 +51,7 @@ def get_package_source( return Source.UNKNOWN -def _sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: +def sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: if len(results) == 0: return [] results.sort(key=lambda y: y[-1]) @@ -59,7 +59,7 @@ def _sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: return list(OrderedDict.fromkeys(results_sorted)) -def _sort_and_deduplicate_tuple( +def sort_and_deduplicate_tuple( results: List[Tuple[str, str, int]] ) -> List[Tuple[str, str]]: if len(results) == 0: @@ -71,12 +71,12 @@ def _sort_and_deduplicate_tuple( return list(OrderedDict.fromkeys(results_sorted)) -def _submodules(mod: pathlib.Path) -> Set[pathlib.Path]: +def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: """Simple submodule finder that doesn't try to import anything""" result = set() if mod.is_dir() and (mod / "__init__.py").exists(): result.add(mod) for child in mod.iterdir(): - result |= _submodules(child) + result |= submodules(child) return result return result From ab11c36923e91fa246c9af1bf17407b8afb283a0 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 4 Apr 2022 22:45:22 -0500 Subject: [PATCH 19/62] add so parsing --- rope/contrib/autoimport/autoimport.py | 17 +++++++---- rope/contrib/autoimport/parse.py | 41 +++++++++++++-------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index a9a362a82..f298cea99 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -10,7 +10,7 @@ from rope.base.resources import Resource from rope.refactor import importutils -from .defs import Name, Source +from .defs import Name, PackageType, Source from .parse import (find_all_names_in_package, get_names_from_builtins, get_names_from_file) from .utils import (get_modname_from_path, get_package_name_from_path, @@ -142,22 +142,24 @@ def generate_modules_cache( """Generate global name cache for external modules listed in `modules`. If no modules are provided, it will generate a cache for every module avalible. This method searches in your sys.path and configured python folders. - Do not use this for generating your own project's internal names, use generate_resource_cache for that instead.""" + Do not use this for generating your own project's internal names, + use generate_resource_cache for that instead.""" job_set = task_handle.create_jobset( "Generating autoimport cache for modules", "all" if modules is None else len(modules), ) packages: List[pathlib.Path] = [] + compiled_packages: List[str] = [] if modules is None: # Get builtins first - self._add_names(get_names_from_builtins(self.underlined)) + compiled_packages.extend(sys.builtin_module_names) folders = self.project.get_python_path_folders() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): package_tuple = get_package_name_from_path(package) if package_tuple is None: continue - package_name = package_tuple[0] + package_name, package_type = package_tuple if ( self.connection.execute( "select * from names where package LIKE (?)", @@ -165,13 +167,14 @@ def generate_modules_cache( ).fetchone() is None ): + if package_type == PackageType.COMPILED: + compiled_packages.append(package_name) packages.append(package) else: for modname in modules: if modname in sys.builtin_module_names: - names = get_names_from_builtins(underlined=True, packages=[modname]) - self._add_names(names) + compiled_packages.append(modname) else: package_path = self._find_package_path(modname) if package_path is None: @@ -186,6 +189,8 @@ def generate_modules_cache( with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(find_all_names_in_package, packages): self._add_names(name_list) + for name_list in exectuor.map(get_names_from_builtins, compiled_packages): + self._add_names(name_list) def update_module(self, module: str): self.generate_modules_cache([module]) diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index a85c7f184..b4870e6fd 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -3,9 +3,8 @@ import ast import inspect import pathlib -import sys from importlib import import_module -from typing import Iterable, List, Tuple +from typing import List, Tuple from .defs import Name, PackageType, Source from .utils import (get_modname_from_path, get_package_name_from_path, @@ -139,27 +138,27 @@ def find_all_names_in_package( def get_names_from_builtins( - underlined: bool = False, packages: Iterable[str] = sys.builtin_module_names + package: str, + underlined: bool = False, ) -> List[Name]: """Gets names from builtin modules. These are the only modules it is safe to get the names from""" + if package == "builtins" or (package.startswith("_") and not underlined): + return [] # Builtins is redundant since you don't have to import it. results: List[Name] = [] - for package in packages: - if package == "builtins": - continue # Builtins is redundant since you don't have to import it. - if underlined or not package.startswith("_"): - module = import_module(package) - if hasattr(module, "__all__"): - for name in module.__all__: + try: + module = import_module(package) + except ImportError: + # print(f"couldn't import {package}") + return [] + if hasattr(module, "__all__"): + for name in module.__all__: + results.append((str(name), package, package, Source.BUILTIN.value)) + for name, value in inspect.getmembers(module): + if underlined or not name.startswith("_"): + if ( + inspect.isclass(value) + or inspect.isfunction(value) + or inspect.isbuiltin(value) + ): results.append((str(name), package, package, Source.BUILTIN.value)) - for name, value in inspect.getmembers(module): - if underlined or not name.startswith("_"): - if ( - inspect.isclass(value) - or inspect.isfunction(value) - or inspect.isbuiltin(value) - ): - results.append( - (str(name), package, package, Source.BUILTIN.value) - ) - return results From 2155385383f01ab3c3c683e070c4327abd3aa086 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 00:10:11 -0500 Subject: [PATCH 20/62] fix builtin handling --- rope/contrib/autoimport/autoimport.py | 42 ++++++++++++++++++--------- rope/contrib/autoimport/defs.py | 4 +-- rope/contrib/autoimport/parse.py | 3 +- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index f298cea99..1f728ff9e 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -129,10 +129,12 @@ def generate_resource_cache( job_set = task_handle.create_jobset( "Generating autoimport cache", len(resources) ) + # Should be very fast, so doesn't need multithreaded computation for file in resources: job_set.started_job("Working on <%s>" % file.path) - self.update_resource(file, underlined) + self.update_resource(file, underlined, commit=False) job_set.finished_job() + self.connection.commit() def generate_modules_cache( self, @@ -173,12 +175,14 @@ def generate_modules_cache( else: for modname in modules: - if modname in sys.builtin_module_names: - compiled_packages.append(modname) + mod_tuple = self._find_package_path(modname) + if mod_tuple is None: + continue + package_path, package_name, package_type = mod_tuple + if package_type in (PackageType.COMPILED, PackageType.BUILTIN): + compiled_packages.append(package_name) else: - package_path = self._find_package_path(modname) - if package_path is None: - continue + assert package_path # Should only return none for a builtin packages.append(package_path) try: packages.remove( @@ -192,6 +196,8 @@ def generate_modules_cache( for name_list in exectuor.map(get_names_from_builtins, compiled_packages): self._add_names(name_list) + self.connection.commit() + def update_module(self, module: str): self.generate_modules_cache([module]) @@ -249,7 +255,9 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - def update_resource(self, resource: Resource, underlined: bool = False): + def update_resource( + self, resource: Resource, underlined: bool = False, commit: bool = True + ): """Update the cache for global names in `resource`""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) @@ -259,6 +267,7 @@ def update_resource(self, resource: Resource, underlined: bool = False): if package_tuple is None: return None package_name = package_tuple[0] + self._del_if_exist(module_name=resource_modname, commit=False) names = get_names_from_file( resource_path, resource_modname, @@ -267,6 +276,8 @@ def update_resource(self, resource: Resource, underlined: bool = False): underlined=underlined, ) self._add_names(names) + if commit: + self.connection.commit() def _changed(self, resource): if not resource.is_folder(): @@ -278,9 +289,10 @@ def _moved(self, resource: Resource, newresource: Resource): self._del_if_exist(modname) self.update_resource(newresource) - def _del_if_exist(self, module_name): + def _del_if_exist(self, module_name, commit: bool = True): self.connection.execute("delete from names where module = ?", (module_name,)) - self.connection.commit() + if commit: + self.connection.commit() @property def _project_name(self): @@ -306,7 +318,6 @@ def _add_names(self, names: List[Name]): "insert into names(name,module,package,source) values (?,?,?,?)", names, ) - self.connection.commit() def _check_import(self, module) -> bool: """ @@ -321,12 +332,17 @@ def _check_all(self): """ pass - def _find_package_path(self, package_name: str) -> Optional[pathlib.Path]: + def _find_package_path( + self, target_name: str + ) -> Optional[Tuple[Optional[pathlib.Path], str, PackageType]]: + if target_name in sys.builtin_module_names: + return (None, target_name, PackageType.BUILTIN) for folder in self.project.get_python_path_folders(): for package in pathlib.Path(folder.path).iterdir(): package_tuple = get_package_name_from_path(package) if package_tuple is None: continue - if package_tuple[0] == package_name: - return package + name, type = package_tuple + if name == target_name: + return (package, name, type) return None diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 63e6fc1b6..b106903d9 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -1,7 +1,6 @@ """Definitions of types for the Autoimport program""" -from typing import Tuple - from enum import Enum +from typing import Tuple class Source(Enum): @@ -17,6 +16,7 @@ class Source(Enum): class PackageType(Enum): + BUILTIN = 0 STANDARD = 1 # Just a folder COMPILED = 2 # .so module SINGLE_FILE = 3 # a .py file diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index b4870e6fd..2d188b582 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -146,13 +146,14 @@ def get_names_from_builtins( return [] # Builtins is redundant since you don't have to import it. results: List[Name] = [] try: - module = import_module(package) + module = import_module(str(package)) except ImportError: # print(f"couldn't import {package}") return [] if hasattr(module, "__all__"): for name in module.__all__: results.append((str(name), package, package, Source.BUILTIN.value)) + else: for name, value in inspect.getmembers(module): if underlined or not name.startswith("_"): if ( From bc21e92f509bd13c57bcb525092ab826dbf7ae7b Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 01:08:18 -0500 Subject: [PATCH 21/62] reformat --- rope/contrib/autoimport/__init__.py | 2 + rope/contrib/autoimport/autoimport.py | 166 +++++++++++++++----------- rope/contrib/autoimport/defs.py | 8 +- rope/contrib/autoimport/parse.py | 88 ++++++++++---- rope/contrib/autoimport/utils.py | 13 +- 5 files changed, 174 insertions(+), 103 deletions(-) diff --git a/rope/contrib/autoimport/__init__.py b/rope/contrib/autoimport/__init__.py index d185a7943..1990bad21 100644 --- a/rope/contrib/autoimport/__init__.py +++ b/rope/contrib/autoimport/__init__.py @@ -1,2 +1,4 @@ +"""AutoImport module for rope.""" from .autoimport import AutoImport + __all__ = ["AutoImport"] diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 1f728ff9e..59c44eead 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -1,3 +1,4 @@ +"""AutoImport module for rope.""" import pathlib import re import sqlite3 @@ -11,14 +12,21 @@ from rope.refactor import importutils from .defs import Name, PackageType, Source -from .parse import (find_all_names_in_package, get_names_from_builtins, - get_names_from_file) -from .utils import (get_modname_from_path, get_package_name_from_path, - sort_and_deduplicate, sort_and_deduplicate_tuple) - - -class AutoImport(object): - """A class for finding the module that provides a name +from .parse import ( + find_all_names_in_package, + get_names_from_compiled, + get_names_from_file, +) +from .utils import ( + get_modname_from_path, + get_package_name_from_path, + sort_and_deduplicate, + sort_and_deduplicate_tuple, +) + + +class AutoImport: + """A class for finding the module that provides a name. This class maintains a cache of global names in python modules. Note that this cache is not accurate and might be out of date. @@ -30,12 +38,16 @@ class AutoImport(object): project: Project def __init__(self, project, observe=True, underlined=False, memory=True): - """Construct an AutoImport object - - If `observe` is `True`, listen for project changes and update - the cache. - - If `underlined` is `True`, underlined names are cached, too. + """Construct an AutoImport object. + + Parameters + ___________ + project : rope.base.project.Project + the project to use for project imports + observe : bool + if true, listen for project changes and update the cache. + underlined : bool + If `underlined` is `True`, underlined names are cached, too. """ self.project = project self.underlined = underlined @@ -50,15 +62,20 @@ def __init__(self, project, observe=True, underlined=False, memory=True): project.add_observer(observer) def _setup_db(self): - self.connection.execute( - "create table if not exists names(name TEXT, module TEXT, package TEXT, source INTEGER)" - ) - - def import_assist(self, starting): - """Return a list of ``(name, module)`` tuples + table = "(name TEXT, module TEXT, package TEXT, source INTEGER)" + self.connection.execute(f"create table if not exists names{table}") - This function tries to find modules that have a global name - that starts with `starting`. + def import_assist(self, starting: str): + """ + Find modules that have a global name that starts with `starting`. + + Parameters + __________ + starting : str + what all the names should start with + Return + __________ + Return a list of ``(name, module)`` tuples """ results = self.connection.execute( "select name, module, source from names WHERE name LIKE (?)", @@ -72,10 +89,10 @@ def import_assist(self, starting): ) # Remove duplicates from multiple occurences of the same item def search(self, name) -> List[str]: - """Searches both modules and names for an import string""" + """Search both modules and names for an import string.""" results: List[Tuple[str, int]] = [] - for name, module, source in self.connection.execute( - "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) + for module, source in self.connection.execute( + "SELECT module, source FROM names WHERE name LIKE (?)", (name,) ): results.append((f"from {module} import {name}", source)) for module, source in self.connection.execute( @@ -91,7 +108,7 @@ def search(self, name) -> List[str]: return sort_and_deduplicate(results) def get_modules(self, name) -> List[str]: - """Return the list of modules that have global `name`""" + """Get the list of modules that have global `name`.""" results = self.connection.execute( "SELECT module, source FROM names WHERE name LIKE (?)", (name,) ).fetchall() @@ -101,13 +118,13 @@ def get_modules(self, name) -> List[str]: return sort_and_deduplicate(results) def get_all_names(self) -> List[str]: - """Return the list of all cached global names""" + """Get the list of all cached global names.""" self._check_all() results = self.connection.execute("select name from names").fetchall() return results def get_all(self) -> List[Name]: - """Dumps the entire database""" + """Dump the entire database.""" self._check_all() results = self.connection.execute("select * from names").fetchall() return results @@ -118,7 +135,7 @@ def generate_resource_cache( underlined: bool = False, task_handle=taskhandle.NullTaskHandle(), ): - """Generate global name cache for project files + """Generate global name cache for project files. If `resources` is a list of `rope.base.resource.File`, only those files are searched; otherwise all python modules in the @@ -131,7 +148,7 @@ def generate_resource_cache( ) # Should be very fast, so doesn't need multithreaded computation for file in resources: - job_set.started_job("Working on <%s>" % file.path) + job_set.started_job(f"Working on {file.path}") self.update_resource(file, underlined, commit=False) job_set.finished_job() self.connection.commit() @@ -141,38 +158,18 @@ def generate_modules_cache( modules: List[str] = None, task_handle=taskhandle.NullTaskHandle(), ): - """Generate global name cache for external modules listed in `modules`. + """ + Generate global name cache for external modules listed in `modules`. + If no modules are provided, it will generate a cache for every module avalible. This method searches in your sys.path and configured python folders. Do not use this for generating your own project's internal names, - use generate_resource_cache for that instead.""" - job_set = task_handle.create_jobset( - "Generating autoimport cache for modules", - "all" if modules is None else len(modules), - ) + use generate_resource_cache for that instead. + """ packages: List[pathlib.Path] = [] compiled_packages: List[str] = [] if modules is None: - # Get builtins first - compiled_packages.extend(sys.builtin_module_names) - folders = self.project.get_python_path_folders() - for folder in folders: - for package in pathlib.Path(folder.path).iterdir(): - package_tuple = get_package_name_from_path(package) - if package_tuple is None: - continue - package_name, package_type = package_tuple - if ( - self.connection.execute( - "select * from names where package LIKE (?)", - (package_name,), - ).fetchone() - is None - ): - if package_type == PackageType.COMPILED: - compiled_packages.append(package_name) - packages.append(package) - + packages, compiled_packages = self._get_avalible_packages() else: for modname in modules: mod_tuple = self._find_package_path(modname) @@ -193,20 +190,23 @@ def generate_modules_cache( with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(find_all_names_in_package, packages): self._add_names(name_list) - for name_list in exectuor.map(get_names_from_builtins, compiled_packages): + for name_list in exectuor.map(get_names_from_compiled, compiled_packages): self._add_names(name_list) self.connection.commit() def update_module(self, module: str): + """Update a module in the cache, or add it if it doesn't exist.""" + self._del_if_exist(module) self.generate_modules_cache([module]) def close(self): + """Close the autoimport database.""" self.connection.commit() self.connection.close() def get_name_locations(self, name): - """Return a list of ``(resource, lineno)`` tuples""" + """Return a list of ``(resource, lineno)`` tuples.""" result = [] names = self.connection.execute( "select module from names where name like (?)", (name,) @@ -227,7 +227,7 @@ def get_name_locations(self, name): return result def clear_cache(self): - """Clear all entries in global-name cache + """Clear all entries in global-name cache. It might be a good idea to use this function before regenerating global names. @@ -238,7 +238,7 @@ def clear_cache(self): self.connection.commit() def find_insertion_line(self, code): - """Guess at what line the new import should be inserted""" + """Guess at what line the new import should be inserted.""" match = re.search(r"^(def|class)\s+", code) if match is not None: code = code[: match.start()] @@ -258,14 +258,14 @@ def find_insertion_line(self, code): def update_resource( self, resource: Resource, underlined: bool = False, commit: bool = True ): - """Update the cache for global names in `resource`""" + """Update the cache for global names in `resource`.""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) resource_modname: str = get_modname_from_path(resource_path, package_path) package_tuple = get_package_name_from_path(package_path) underlined = underlined if underlined else self.underlined if package_tuple is None: - return None + return package_name = package_tuple[0] self._del_if_exist(module_name=resource_modname, commit=False) names = get_names_from_file( @@ -294,6 +294,29 @@ def _del_if_exist(self, module_name, commit: bool = True): if commit: self.connection.commit() + def _get_avalible_packages(self) -> Tuple[List[pathlib.Path], List[str]]: + packages: List[pathlib.Path] = [] + # Get builtins first + compiled_packages: List[str] = list(sys.builtin_module_names) + folders = self.project.get_python_path_folders() + for folder in folders: + for package in pathlib.Path(folder.path).iterdir(): + package_tuple = get_package_name_from_path(package) + if package_tuple is None: + continue + package_name, package_type = package_tuple + if ( + self.connection.execute( + "select * from names where package LIKE (?)", + (package_name,), + ).fetchone() + is None + ): + if package_type == PackageType.COMPILED: + compiled_packages.append(package_name) + packages.append(package) + return packages, compiled_packages + @property def _project_name(self): package_path: pathlib.Path = pathlib.Path(self.project.address) @@ -319,17 +342,22 @@ def _add_names(self, names: List[Name]): names, ) - def _check_import(self, module) -> bool: + def _check_import(self, module: pathlib.Path) -> bool: """ - Checks the ability to import an external package, removes it if not avalible + Check the ability to import an external package, removes it if not avalible. + + Parameters + ---------- + module: pathlib.path + The module to check + Returns + ---------- """ # Not Implemented Yet, silently will fail return True def _check_all(self): - """ - Checks all modules and removes bad ones - """ + """Check all modules and removes bad ones.""" pass def _find_package_path( @@ -342,7 +370,7 @@ def _find_package_path( package_tuple = get_package_name_from_path(package) if package_tuple is None: continue - name, type = package_tuple + name, package_type = package_tuple if name == target_name: - return (package, name, type) + return (package, name, package_type) return None diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index b106903d9..b1eb9aab5 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -1,9 +1,11 @@ -"""Definitions of types for the Autoimport program""" +"""Definitions of types for the Autoimport program.""" from enum import Enum from typing import Tuple class Source(Enum): + """Describes the source of the package, for sorting purposes.""" + PROJECT = 0 # Obviously any project packages come first MANUAL = 1 # Any packages manually added are probably important to the user BUILTIN = 2 @@ -16,7 +18,9 @@ class Source(Enum): class PackageType(Enum): - BUILTIN = 0 + """Describes the type of package, to determine how to get the names from it.""" + + BUILTIN = 0 # No file exists, compiled into python. IE: Sys STANDARD = 1 # Just a folder COMPILED = 2 # .so module SINGLE_FILE = 3 # a .py file diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 2d188b582..c2892abd3 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -1,4 +1,8 @@ -"""Files to parse the source code of a python file, so object, or builtin module for autoimport names""" +""" +Functions to find importable names. + +Can extract names from source code of a python file, .so object, or builtin module. +""" import ast import inspect @@ -18,11 +22,10 @@ def get_names( package_source: Source, underlined: bool = False, ) -> List[Name]: - """Update the cache for global names in `modname` module + """Get all names in the `modname` module, located at modpath. `modname` is the name of a module. """ - # TODO use __all__ parsing if avalible if modpath.is_dir(): names: List[Name] if modpath / "__init__.py": @@ -47,10 +50,27 @@ def get_names( ) ) return names - else: - return get_names_from_file( - modpath, modname, package_name, package_source, underlined=underlined + return get_names_from_file( + modpath, modname, package_name, package_source, underlined=underlined + ) + + +def parse_all(node: ast.Assign, modname: str, package: str, package_source: Source): + """Parse the node which contains the value __all__ and return its contents.""" + # I assume that the __all__ value isn't assigned via tuple + all_results: List[Name] = [] + assert isinstance(node.value, ast.List) + for item in node.value.elts: + assert isinstance(item, ast.Constant) + all_results.append( + ( + str(item.value), + modname, + package, + package_source.value, + ) ) + return all_results def get_names_from_file( @@ -61,11 +81,19 @@ def get_names_from_file( only_all: bool = False, underlined: bool = False, ) -> List[Name]: + """ + Get all the names from a given file using ast. + + Parameters + __________ + only_all: bool + only use __all__ to determine the module's contents + """ with open(module, mode="rb") as file: try: root_node = ast.parse(file.read()) - except SyntaxError as e: - print(e) + except SyntaxError as error: + print(error) return [] results: List[Name] = [] for node in ast.iter_child_nodes(root_node): @@ -75,22 +103,8 @@ def get_names_from_file( try: assert isinstance(target, ast.Name) if target.id == "__all__": - # TODO add tuple handling - all_results: List[Name] = [] - assert isinstance(node.value, ast.List) - for item in node.value.elts: - assert isinstance(item, ast.Constant) - all_results.append( - ( - str(item.value), - modname, - package, - package_source.value, - ) - ) - return all_results - else: - node_names.append(target.id) + return parse_all(node, modname, package, package_source) + node_names.append(target.id) except (AttributeError, AssertionError): # TODO handle tuple assignment pass @@ -110,6 +124,18 @@ def find_all_names_in_package( package_source: Source = None, underlined: bool = False, ) -> List[Name]: + """ + Find all names in a package. + + Parameters + ---------- + package_path : pathlib.Path + path to the package + recursive : bool + scan submodules in addition to the root directory + underlined : bool + include underlined directories + """ package_tuple = get_package_name_from_path(package_path) if package_tuple is None: return [] @@ -137,11 +163,21 @@ def find_all_names_in_package( return result -def get_names_from_builtins( +def get_names_from_compiled( package: str, underlined: bool = False, ) -> List[Name]: - """Gets names from builtin modules. These are the only modules it is safe to get the names from""" + """ + Get the names from a compiled module. + + Instead of using ast, it imports the module. + Parameters + ---------- + package : str + package to import. Must be in sys.path + underlined : bool + include underlined names + """ if package == "builtins" or (package.startswith("_") and not underlined): return [] # Builtins is redundant since you don't have to import it. results: List[Name] = [] diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 23a7521a6..021b1cef5 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -1,4 +1,4 @@ -"""Utility functions for the autoimport code""" +"""Utility functions for the autoimport code.""" import pathlib import sys from collections import OrderedDict @@ -12,13 +12,13 @@ def get_package_name_from_path( package_path: pathlib.Path, ) -> Optional[Tuple[str, PackageType]]: + """Get package name and type from a path.""" package_name = package_path.name if package_name.endswith(".egg-info"): return None if package_name.endswith(".so"): name = package_name.split(".")[0] return (name, PackageType.COMPILED) - # TODO add so handling if package_name.endswith(".py"): stripped_name = package_name.removesuffix(".py") return (stripped_name, PackageType.SINGLE_FILE) @@ -26,6 +26,7 @@ def get_package_name_from_path( def get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: + """Get module name from a path in respect to package.""" package_name: str = package_path.name modname = ( modpath.relative_to(package_path) @@ -47,11 +48,11 @@ def get_package_source( return Source.SITE_PACKAGE if package.as_posix().startswith(sys.prefix): return Source.STANDARD - else: - return Source.UNKNOWN + return Source.UNKNOWN def sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: + """Sort and deduplicate a list of name, source entries.""" if len(results) == 0: return [] results.sort(key=lambda y: y[-1]) @@ -62,6 +63,7 @@ def sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: def sort_and_deduplicate_tuple( results: List[Tuple[str, str, int]] ) -> List[Tuple[str, str]]: + """Sort and deduplicate a list of name, module, source entries.""" if len(results) == 0: return [] results.sort(key=lambda y: y[-1]) @@ -72,11 +74,10 @@ def sort_and_deduplicate_tuple( def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: - """Simple submodule finder that doesn't try to import anything""" + """Find submodules in a given path using __init__.py.""" result = set() if mod.is_dir() and (mod / "__init__.py").exists(): result.add(mod) for child in mod.iterdir(): result |= submodules(child) return result - return result From 03585be3b97fd5cfbd2d739dd005f30ec49c765f Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 01:33:02 -0500 Subject: [PATCH 22/62] fix project location issue --- rope/contrib/autoimport/autoimport.py | 40 ++++++++++++++++---------- rope/contrib/autoimport/utils.py | 10 +++++-- ropetest/contrib/autoimporttest.py | 41 ++++++++++++--------------- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 59c44eead..c85555366 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -12,17 +12,10 @@ from rope.refactor import importutils from .defs import Name, PackageType, Source -from .parse import ( - find_all_names_in_package, - get_names_from_compiled, - get_names_from_file, -) -from .utils import ( - get_modname_from_path, - get_package_name_from_path, - sort_and_deduplicate, - sort_and_deduplicate_tuple, -) +from .parse import (find_all_names_in_package, get_names_from_compiled, + get_names_from_file) +from .utils import (get_modname_from_path, get_package_name_from_path, + sort_and_deduplicate, sort_and_deduplicate_tuple) class AutoImport: @@ -48,6 +41,8 @@ def __init__(self, project, observe=True, underlined=False, memory=True): if true, listen for project changes and update the cache. underlined : bool If `underlined` is `True`, underlined names are cached, too. + memory : bool + if true, don't persist to disk """ self.project = project self.underlined = underlined @@ -171,6 +166,12 @@ def generate_modules_cache( if modules is None: packages, compiled_packages = self._get_avalible_packages() else: + try: + modules.remove( + self._project_name + ) # Don't want to generate the cache for the user's project + except ValueError: + pass for modname in modules: mod_tuple = self._find_package_path(modname) if mod_tuple is None: @@ -183,7 +184,7 @@ def generate_modules_cache( packages.append(package_path) try: packages.remove( - self._project_name + self._project_path ) # Don't want to generate the cache for the user's project except ValueError: pass @@ -260,8 +261,13 @@ def update_resource( ): """Update the cache for global names in `resource`.""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) - package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = get_modname_from_path(resource_path, package_path) + package_path: pathlib.Path = self._project_path + # The project doesn't need its name added to the path, + # since the standard python file layout accounts for that + # so we set add_package_name to False + resource_modname: str = get_modname_from_path( + resource_path, package_path, add_package_name=False + ) package_tuple = get_package_name_from_path(package_path) underlined = underlined if underlined else self.underlined if package_tuple is None: @@ -325,10 +331,14 @@ def _project_name(self): return None return package_tuple[0] + @property + def _project_path(self): + return pathlib.Path(self.project.address) + def _modname(self, resource: Resource): resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = get_modname_from_path(resource_path, package_path) + resource_modname: str = get_modname_from_path(resource_path, package_path, add_package_name=False) return resource_modname def _removed(self, resource): diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 021b1cef5..96c717dc2 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -25,16 +25,22 @@ def get_package_name_from_path( return (package_name, PackageType.STANDARD) -def get_modname_from_path(modpath: pathlib.Path, package_path: pathlib.Path) -> str: +def get_modname_from_path( + modpath: pathlib.Path, package_path: pathlib.Path, add_package_name: bool = True +) -> str: """Get module name from a path in respect to package.""" package_name: str = package_path.name modname = ( modpath.relative_to(package_path) .as_posix() + .removesuffix("/__init__.py") .removesuffix(".py") .replace("/", ".") ) - modname = package_name if modname == "." else package_name + "." + modname + if add_package_name: + modname = package_name if modname == "." else package_name + "." + modname + else: + assert modname != "." return modname diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 0ad197da0..c5d211fc6 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -12,11 +12,6 @@ def project(): testutils.remove_project(project) -@pytest.fixture -def project_name(): - return "sample_project" - - @pytest.fixture def importer(project): importer = autoimport.AutoImport(project, observe=False, memory=True) @@ -53,12 +48,12 @@ class TestAutoImport: def test_simple_case(self, importer): assert [] == importer.import_assist("A") - def test_update_resource(self, importer, mod1, project_name): + def test_update_resource(self, importer, mod1): mod1.write("myvar = None\n") importer.update_resource(mod1) - assert [("myvar", f"{project_name}.mod1")] == importer.import_assist("myva") + assert [("myvar", f"mod1")] == importer.import_assist("myva") - def test_update_module(self, importer, mod1, project_name): + def test_update_module(self, importer, mod1): mod1.write("myvar = None") importer.update_module("mod1") assert [("myvar", "mod1")] == importer.import_assist("myva") @@ -77,19 +72,19 @@ def test_excluding_imported_names(self, mod1, importer): importer.update_resource(mod1) assert [] == importer.import_assist("pkg") - def test_get_modules(self, mod1, importer, project_name): + def test_get_modules(self, mod1, importer): mod1.write("myvar = None\n") importer.update_resource(mod1) - assert [f"{project_name}.mod1"] == importer.get_modules("myvar") + assert ["mod1"] == importer.get_modules("myvar") - def test_get_modules_inside_packages(self, mod1, mod2, importer, project_name): + def test_get_modules_inside_packages(self, mod1, mod2, importer): mod1.write("myvar = None\n") mod2.write("myvar = None\n") importer.update_resource(mod1) importer.update_resource(mod2) assert [ - f"{project_name}.mod1", - f"{project_name}.pkg.mod2", + "mod1", + "pkg.mod2", ] == importer.get_modules("myvar") def test_trivial_insertion_line(self, importer): @@ -112,28 +107,28 @@ def test_insertion_line_with_blank_lines(self, importer): result = importer.find_insertion_line("import mod1\n\n# comment\n") assert 2 == result - def test_empty_cache(self, importer, mod1, project_name): + def test_empty_cache(self, importer, mod1): mod1.write("myvar = None\n") importer.update_resource(mod1) - assert [f"{project_name}.mod1"] == importer.get_modules("myvar") + assert ["mod1"] == importer.get_modules("myvar") importer.clear_cache() assert [] == importer.get_modules("myvar") - def test_not_caching_underlined_names(self, importer, mod1, project_name): + def test_not_caching_underlined_names(self, importer, mod1): mod1.write("_myvar = None\n") importer.update_resource(mod1, underlined=False) assert [] == importer.get_modules("_myvar") importer.update_resource(mod1, underlined=True) - assert [f"{project_name}.mod1"] == importer.get_modules("_myvar") + assert ["mod1"] == importer.get_modules("_myvar") def test_caching_underlined_names_passing_to_the_constructor( - self, project, mod1, project_name + self, project, mod1 ): importer = autoimport.AutoImport(project, False, True) mod1.write("_myvar = None\n") importer.update_resource(mod1) - assert [f"{project_name}.mod1"] == importer.get_modules("_myvar") + assert ["mod1"] == importer.get_modules("_myvar") importer.close() def test_name_locations(self, importer, mod1): @@ -154,14 +149,14 @@ def test_handling_builtin_modules(self, importer): class TestAutoImportObserving: - def test_writing_files(self, importer_observing, mod1, project_name): + def test_writing_files(self, importer_observing, mod1): mod1.write("myvar = None\n") - assert [f"{project_name}.mod1"] == importer_observing.get_modules("myvar") + assert ["mod1"] == importer_observing.get_modules("myvar") - def test_moving_files(self, importer_observing, mod1, project_name): + def test_moving_files(self, importer_observing, mod1): mod1.write("myvar = None\n") mod1.move("mod3.py") - assert [f"{project_name}.mod3"] == importer_observing.get_modules("myvar") + assert ["mod3"] == importer_observing.get_modules("myvar") def test_removing_files(self, importer_observing, mod1): mod1.write("myvar = None\n") From ebfc3b8c1127cf8534802a01e89f98fca7ce6839 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 01:58:30 -0500 Subject: [PATCH 23/62] update documentation --- CHANGELOG.md | 3 ++- docs/library.rst | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e1dcfc9e..b78ac46d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # **Upcoming release** - +## New feature +- #464 Improve autoimport code to use a sqllite3 database, cache all available modules quickly, search for names and produce import statements, sort import statements. ## Bug fixes - #459 Fix bug while extracting method with augmented assignment to subscript in try block (@dryobates) diff --git a/docs/library.rst b/docs/library.rst index 60f6303e6..b1441175d 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -834,6 +834,20 @@ can use this module to auto-import names. ``AutoImport.get_modules()`` returns the list of modules with the given global name. ``AutoImport.import_assist()`` tries to find the modules that have a global name that starts with the given prefix. +It uses an sqllite3 database, which can be made persistent by passing memory as false to the constructor. +It must be closed when done with the ```AutoImport.close()``` method +It can search for a name from both modules and statements you can import from them +```py +from rope.base.project import Project +from rope.contrib.autoimport import AutoImport + +project = Project("/path/to/project") +autoimport = AutoImport(project, memory=False) +autoimport.generate_resource_cache() # Generates a cache of the local modules, from the project you're working on +autoimport.generate_modules_cache() # Generates a cache of external modules +print(autoimport.search("AutoImport")) +autoimport.close() +``` Cross-Project Refactorings From f719f9523a2a80b0498d7e81b18398e3d9e30e1f Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 02:00:29 -0500 Subject: [PATCH 24/62] use code block --- docs/library.rst | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/library.rst b/docs/library.rst index b1441175d..6d541f696 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -837,17 +837,15 @@ global name that starts with the given prefix. It uses an sqllite3 database, which can be made persistent by passing memory as false to the constructor. It must be closed when done with the ```AutoImport.close()``` method It can search for a name from both modules and statements you can import from them -```py -from rope.base.project import Project -from rope.contrib.autoimport import AutoImport - -project = Project("/path/to/project") -autoimport = AutoImport(project, memory=False) -autoimport.generate_resource_cache() # Generates a cache of the local modules, from the project you're working on -autoimport.generate_modules_cache() # Generates a cache of external modules -print(autoimport.search("AutoImport")) -autoimport.close() -``` +.. code-block:: python + from rope.base.project import Project + from rope.contrib.autoimport import AutoImport + project = Project("/path/to/project") + autoimport = AutoImport(project, memory=False) + autoimport.generate_resource_cache() # Generates a cache of the local modules, from the project you're working on + autoimport.generate_modules_cache() # Generates a cache of external modules + print(autoimport.search("AutoImport")) + autoimport.close() Cross-Project Refactorings From 90a1600b7e86e293637aa53931d155575371d1e1 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 02:01:41 -0500 Subject: [PATCH 25/62] use code block --- docs/library.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/library.rst b/docs/library.rst index 6d541f696..2671a2718 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -837,9 +837,12 @@ global name that starts with the given prefix. It uses an sqllite3 database, which can be made persistent by passing memory as false to the constructor. It must be closed when done with the ```AutoImport.close()``` method It can search for a name from both modules and statements you can import from them -.. code-block:: python + +.. code-block:: python + from rope.base.project import Project from rope.contrib.autoimport import AutoImport + project = Project("/path/to/project") autoimport = AutoImport(project, memory=False) autoimport.generate_resource_cache() # Generates a cache of the local modules, from the project you're working on From a9ddf65a538485a9feb7087b3cf53386253c38df Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 02:03:46 -0500 Subject: [PATCH 26/62] improve formatting and change example to Dict --- docs/library.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/library.rst b/docs/library.rst index 2671a2718..a4031f375 100644 --- a/docs/library.rst +++ b/docs/library.rst @@ -834,9 +834,11 @@ can use this module to auto-import names. ``AutoImport.get_modules()`` returns the list of modules with the given global name. ``AutoImport.import_assist()`` tries to find the modules that have a global name that starts with the given prefix. + It uses an sqllite3 database, which can be made persistent by passing memory as false to the constructor. -It must be closed when done with the ```AutoImport.close()``` method -It can search for a name from both modules and statements you can import from them +It must be closed when done with the ```AutoImport.close()``` method. + +AutoImport can search for a name from both modules and statements you can import from them. .. code-block:: python @@ -847,7 +849,7 @@ It can search for a name from both modules and statements you can import from th autoimport = AutoImport(project, memory=False) autoimport.generate_resource_cache() # Generates a cache of the local modules, from the project you're working on autoimport.generate_modules_cache() # Generates a cache of external modules - print(autoimport.search("AutoImport")) + print(autoimport.search("Dict")) autoimport.close() From ed12e2055a65f51540ace9703eae4836e8758c87 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 02:18:33 -0500 Subject: [PATCH 27/62] use old testsuite(mostly) --- rope/contrib/autoimport/autoimport.py | 16 +- ropetest/contrib/autoimporttest.py | 304 ++++++++++++-------------- 2 files changed, 151 insertions(+), 169 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index c85555366..043c2d55a 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -165,13 +165,13 @@ def generate_modules_cache( compiled_packages: List[str] = [] if modules is None: packages, compiled_packages = self._get_avalible_packages() - else: try: - modules.remove( - self._project_name + packages.remove( + self._project_path ) # Don't want to generate the cache for the user's project except ValueError: pass + else: for modname in modules: mod_tuple = self._find_package_path(modname) if mod_tuple is None: @@ -182,12 +182,6 @@ def generate_modules_cache( else: assert package_path # Should only return none for a builtin packages.append(package_path) - try: - packages.remove( - self._project_path - ) # Don't want to generate the cache for the user's project - except ValueError: - pass with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(find_all_names_in_package, packages): self._add_names(name_list) @@ -338,7 +332,9 @@ def _project_path(self): def _modname(self, resource: Resource): resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = get_modname_from_path(resource_path, package_path, add_package_name=False) + resource_modname: str = get_modname_from_path( + resource_path, package_path, add_package_name=False + ) return resource_modname def _removed(self, resource): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index c5d211fc6..dc183a1f4 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -1,164 +1,150 @@ -import pytest +try: + import unittest2 as unittest +except ImportError: + import unittest from rope.contrib import autoimport from ropetest import testutils -@pytest.fixture -def project(): - project = testutils.sample_project(extension_modules=["sys"]) - project.set("python_path", [project.address]) - yield project - testutils.remove_project(project) - - -@pytest.fixture -def importer(project): - importer = autoimport.AutoImport(project, observe=False, memory=True) - yield importer - importer.close() - - -@pytest.fixture -def mod1(importer, project): - mod1 = testutils.create_module(project, "mod1") - yield mod1 - - -@pytest.fixture -def pkg(importer, project): - pkg = testutils.create_package(project, "pkg") - yield pkg - - -@pytest.fixture -def importer_observing(project): - importer = autoimport.AutoImport(project, observe=True, memory=True) - yield importer - importer.close() - - -@pytest.fixture -def mod2(importer, project, pkg): - mod2 = testutils.create_module(project, "mod2", pkg) - yield mod2 - - -class TestAutoImport: - def test_simple_case(self, importer): - assert [] == importer.import_assist("A") - - def test_update_resource(self, importer, mod1): - mod1.write("myvar = None\n") - importer.update_resource(mod1) - assert [("myvar", f"mod1")] == importer.import_assist("myva") - - def test_update_module(self, importer, mod1): - mod1.write("myvar = None") - importer.update_module("mod1") - assert [("myvar", "mod1")] == importer.import_assist("myva") - - def test_update_non_existent_module(self, importer): - importer.update_module("does_not_exists_this") - assert [] == importer.import_assist("myva") - - def test_module_with_syntax_errors(self, importer, mod1): - mod1.write("this is a syntax error\n") - importer.update_resource(mod1) - assert [] == importer.import_assist("myva") - - def test_excluding_imported_names(self, mod1, importer): - mod1.write("import pkg\n") - importer.update_resource(mod1) - assert [] == importer.import_assist("pkg") - - def test_get_modules(self, mod1, importer): - mod1.write("myvar = None\n") - importer.update_resource(mod1) - assert ["mod1"] == importer.get_modules("myvar") - - def test_get_modules_inside_packages(self, mod1, mod2, importer): - mod1.write("myvar = None\n") - mod2.write("myvar = None\n") - importer.update_resource(mod1) - importer.update_resource(mod2) - assert [ - "mod1", - "pkg.mod2", - ] == importer.get_modules("myvar") - - def test_trivial_insertion_line(self, importer): - result = importer.find_insertion_line("") - assert 1 == result - - def test_insertion_line(self, importer): - result = importer.find_insertion_line("import mod\n") - assert 2 == result - - def test_insertion_line_with_pydocs(self, importer): - result = importer.find_insertion_line('"""docs\n\ndocs"""\nimport mod\n') - assert 5 == result - - def test_insertion_line_with_multiple_imports(self, importer): - result = importer.find_insertion_line("import mod1\n\nimport mod2\n") - assert 4 == result - - def test_insertion_line_with_blank_lines(self, importer): - result = importer.find_insertion_line("import mod1\n\n# comment\n") - assert 2 == result - - def test_empty_cache(self, importer, mod1): - mod1.write("myvar = None\n") - importer.update_resource(mod1) - assert ["mod1"] == importer.get_modules("myvar") - - importer.clear_cache() - assert [] == importer.get_modules("myvar") - - def test_not_caching_underlined_names(self, importer, mod1): - mod1.write("_myvar = None\n") - importer.update_resource(mod1, underlined=False) - assert [] == importer.get_modules("_myvar") - importer.update_resource(mod1, underlined=True) - assert ["mod1"] == importer.get_modules("_myvar") - - def test_caching_underlined_names_passing_to_the_constructor( - self, project, mod1 - ): - importer = autoimport.AutoImport(project, False, True) - mod1.write("_myvar = None\n") - importer.update_resource(mod1) - assert ["mod1"] == importer.get_modules("_myvar") - importer.close() - - def test_name_locations(self, importer, mod1): - mod1.write("myvar = None\n") - importer.update_resource(mod1) - assert [(mod1, 1)] == importer.get_name_locations("myvar") - - def test_name_locations_with_multiple_occurrences(self, mod1, mod2, importer): - mod1.write("myvar = None\n") - mod2.write("\nmyvar = None\n") - importer.update_resource(mod1) - importer.update_resource(mod2) - assert [(mod1, 1), (mod2, 2)] == importer.get_name_locations("myvar") - - def test_handling_builtin_modules(self, importer): - importer.update_module("sys") - assert ["sys"] == importer.get_modules("exit") - - -class TestAutoImportObserving: - def test_writing_files(self, importer_observing, mod1): - mod1.write("myvar = None\n") - assert ["mod1"] == importer_observing.get_modules("myvar") - - def test_moving_files(self, importer_observing, mod1): - mod1.write("myvar = None\n") - mod1.move("mod3.py") - assert ["mod3"] == importer_observing.get_modules("myvar") - - def test_removing_files(self, importer_observing, mod1): - mod1.write("myvar = None\n") - mod1.remove() - assert [] == importer_observing.get_modules("myvar") +class AutoImportTest(unittest.TestCase): + def setUp(self): + super(AutoImportTest, self).setUp() + self.project = testutils.sample_project(extension_modules=["sys"]) + self.mod1 = testutils.create_module(self.project, "mod1") + self.pkg = testutils.create_package(self.project, "pkg") + self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) + self.importer = autoimport.AutoImport(self.project, observe=False) + + def tearDown(self): + testutils.remove_project(self.project) + super(AutoImportTest, self).tearDown() + + def test_simple_case(self): + self.assertEqual([], self.importer.import_assist("A")) + + def test_update_resource(self): + self.mod1.write("myvar = None\n") + self.importer.update_resource(self.mod1) + self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) + + # def test_update_module(self): + # update module only works on external modules. + # self.mod1.write("myvar = None") + # self.importer.update_module("rope") + # self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) + + def test_update_non_existent_module(self): + self.importer.update_module("does_not_exists_this") + self.assertEqual([], self.importer.import_assist("myva")) + + def test_module_with_syntax_errors(self): + self.mod1.write("this is a syntax error\n") + self.importer.update_resource(self.mod1) + self.assertEqual([], self.importer.import_assist("myva")) + + def test_excluding_imported_names(self): + self.mod1.write("import pkg\n") + self.importer.update_resource(self.mod1) + self.assertEqual([], self.importer.import_assist("pkg")) + + def test_get_modules(self): + self.mod1.write("myvar = None\n") + self.importer.update_resource(self.mod1) + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + + def test_get_modules_inside_packages(self): + self.mod1.write("myvar = None\n") + self.mod2.write("myvar = None\n") + self.importer.update_resource(self.mod1) + self.importer.update_resource(self.mod2) + self.assertEqual( + set(["mod1", "pkg.mod2"]), set(self.importer.get_modules("myvar")) + ) + + def test_trivial_insertion_line(self): + result = self.importer.find_insertion_line("") + self.assertEqual(1, result) + + def test_insertion_line(self): + result = self.importer.find_insertion_line("import mod\n") + self.assertEqual(2, result) + + def test_insertion_line_with_pydocs(self): + result = self.importer.find_insertion_line('"""docs\n\ndocs"""\nimport mod\n') + self.assertEqual(5, result) + + def test_insertion_line_with_multiple_imports(self): + result = self.importer.find_insertion_line("import mod1\n\nimport mod2\n") + self.assertEqual(4, result) + + def test_insertion_line_with_blank_lines(self): + result = self.importer.find_insertion_line("import mod1\n\n# comment\n") + self.assertEqual(2, result) + + def test_empty_cache(self): + self.mod1.write("myvar = None\n") + self.importer.update_resource(self.mod1) + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + self.importer.clear_cache() + self.assertEqual([], self.importer.get_modules("myvar")) + + def test_not_caching_underlined_names(self): + self.mod1.write("_myvar = None\n") + self.importer.update_resource(self.mod1, underlined=False) + self.assertEqual([], self.importer.get_modules("_myvar")) + self.importer.update_resource(self.mod1, underlined=True) + self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) + + def test_caching_underlined_names_passing_to_the_constructor(self): + importer = autoimport.AutoImport(self.project, False, True) + self.mod1.write("_myvar = None\n") + importer.update_resource(self.mod1) + self.assertEqual(["mod1"], importer.get_modules("_myvar")) + + def test_name_locations(self): + self.mod1.write("myvar = None\n") + self.importer.update_resource(self.mod1) + self.assertEqual([(self.mod1, 1)], self.importer.get_name_locations("myvar")) + + def test_name_locations_with_multiple_occurrences(self): + self.mod1.write("myvar = None\n") + self.mod2.write("\nmyvar = None\n") + self.importer.update_resource(self.mod1) + self.importer.update_resource(self.mod2) + self.assertEqual( + set([(self.mod1, 1), (self.mod2, 2)]), + set(self.importer.get_name_locations("myvar")), + ) + + def test_handling_builtin_modules(self): + self.importer.update_module("sys") + self.assertTrue("sys" in self.importer.get_modules("exit")) + + +class AutoImportObservingTest(unittest.TestCase): + def setUp(self): + super(AutoImportObservingTest, self).setUp() + self.project = testutils.sample_project() + self.mod1 = testutils.create_module(self.project, "mod1") + self.pkg = testutils.create_package(self.project, "pkg") + self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) + self.importer = autoimport.AutoImport(self.project, observe=True) + + def tearDown(self): + testutils.remove_project(self.project) + super(AutoImportObservingTest, self).tearDown() + + def test_writing_files(self): + self.mod1.write("myvar = None\n") + self.assertEqual(["mod1"], self.importer.get_modules("myvar")) + + def test_moving_files(self): + self.mod1.write("myvar = None\n") + self.mod1.move("mod3.py") + self.assertEqual(["mod3"], self.importer.get_modules("myvar")) + + def test_removing_files(self): + self.mod1.write("myvar = None\n") + self.mod1.remove() + self.assertEqual([], self.importer.get_modules("myvar")) From b197ce4db1bb3d021b2ce5b4b858bd19b864aad2 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 16:10:09 -0500 Subject: [PATCH 28/62] move packages to dedicated database and improve detection of bogus packages --- rope/contrib/autoimport/autoimport.py | 66 +++++++++++++++++---------- rope/contrib/autoimport/defs.py | 2 +- rope/contrib/autoimport/utils.py | 23 ++++++---- 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 043c2d55a..99f6c2781 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -4,6 +4,7 @@ import sqlite3 import sys from concurrent.futures import ProcessPoolExecutor +from itertools import chain from typing import List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle @@ -11,7 +12,7 @@ from rope.base.resources import Resource from rope.refactor import importutils -from .defs import Name, PackageType, Source +from .defs import Name, Package, PackageType, Source from .parse import (find_all_names_in_package, get_names_from_compiled, get_names_from_file) from .utils import (get_modname_from_path, get_package_name_from_path, @@ -57,8 +58,11 @@ def __init__(self, project, observe=True, underlined=False, memory=True): project.add_observer(observer) def _setup_db(self): - table = "(name TEXT, module TEXT, package TEXT, source INTEGER)" - self.connection.execute(f"create table if not exists names{table}") + packages_table = "(pacakge TEXT)" + names_table = "(name TEXT, module TEXT, package TEXT, source INTEGER)" + self.connection.execute(f"create table if not exists names{names_table}") + self.connection.execute(f"create table if not exists packages{packages_table}") + self.connection.commit() def import_assist(self, starting: str): """ @@ -118,11 +122,12 @@ def get_all_names(self) -> List[str]: results = self.connection.execute("select name from names").fetchall() return results - def get_all(self) -> List[Name]: + def _dump_all(self) -> Tuple[List[Name], List[Package]]: """Dump the entire database.""" self._check_all() - results = self.connection.execute("select * from names").fetchall() - return results + name_results = self.connection.execute("select * from names").fetchall() + package_results = self.connection.execute("select * from packages").fetchall() + return name_results, package_results def generate_resource_cache( self, @@ -163,25 +168,25 @@ def generate_modules_cache( """ packages: List[pathlib.Path] = [] compiled_packages: List[str] = [] + to_add: List[Package] = [] if modules is None: - packages, compiled_packages = self._get_avalible_packages() - try: - packages.remove( - self._project_path - ) # Don't want to generate the cache for the user's project - except ValueError: - pass + packages, compiled_packages, to_add = self._get_avalible_packages() else: + existing = self._get_existing() for modname in modules: mod_tuple = self._find_package_path(modname) if mod_tuple is None: continue package_path, package_name, package_type = mod_tuple + if package_name in existing: + continue if package_type in (PackageType.COMPILED, PackageType.BUILTIN): compiled_packages.append(package_name) else: assert package_path # Should only return none for a builtin packages.append(package_path) + to_add.append((package_name,)) + self._add_packages(to_add) with ProcessPoolExecutor() as exectuor: for name_list in exectuor.map(find_all_names_in_package, packages): self._add_names(name_list) @@ -294,28 +299,41 @@ def _del_if_exist(self, module_name, commit: bool = True): if commit: self.connection.commit() - def _get_avalible_packages(self) -> Tuple[List[pathlib.Path], List[str]]: + def _get_avalible_packages( + self, + ) -> Tuple[List[pathlib.Path], List[str], List[Package]]: packages: List[pathlib.Path] = [] + package_names: List[ + Package + ] = [] # List of packages to add to the package table # Get builtins first compiled_packages: List[str] = list(sys.builtin_module_names) folders = self.project.get_python_path_folders() + existing = self._get_existing() for folder in folders: for package in pathlib.Path(folder.path).iterdir(): package_tuple = get_package_name_from_path(package) if package_tuple is None: continue package_name, package_type = package_tuple - if ( - self.connection.execute( - "select * from names where package LIKE (?)", - (package_name,), - ).fetchone() - is None - ): - if package_type == PackageType.COMPILED: - compiled_packages.append(package_name) + if package_name in existing: + continue + if package_type == PackageType.COMPILED: + compiled_packages.append(package_name) + else: packages.append(package) - return packages, compiled_packages + package_names.append((package_name,)) + return packages, compiled_packages, package_names + + def _add_packages(self, packages: List[Package]): + self.connection.executemany("INSERT into packages values(?)", packages) + + def _get_existing(self) -> List[str]: + existing: List[str] = list( + chain(*self.connection.execute("select * from packages").fetchall()) + ) + existing.append(self._project_name) + return existing @property def _project_name(self): diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index b1eb9aab5..efa7d7c93 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -15,7 +15,7 @@ class Source(Enum): Name = Tuple[str, str, str, int] - +Package = Tuple[str] class PackageType(Enum): """Describes the type of package, to determine how to get the names from it.""" diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 96c717dc2..669ab438e 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -12,16 +12,23 @@ def get_package_name_from_path( package_path: pathlib.Path, ) -> Optional[Tuple[str, PackageType]]: - """Get package name and type from a path.""" + """ + Get package name and type from a path. + + Checks for common issues, such as not being a viable python module + Returns None if not a viable package. + """ package_name = package_path.name - if package_name.endswith(".egg-info"): + if package_path.is_file(): + if package_name.endswith(".so"): + name = package_name.split(".")[0] + return (name, PackageType.COMPILED) + if package_name.endswith(".py"): + stripped_name = package_name.removesuffix(".py") + return (stripped_name, PackageType.SINGLE_FILE) + return None + if package_name.endswith((".egg-info", ".dist-info")): return None - if package_name.endswith(".so"): - name = package_name.split(".")[0] - return (name, PackageType.COMPILED) - if package_name.endswith(".py"): - stripped_name = package_name.removesuffix(".py") - return (stripped_name, PackageType.SINGLE_FILE) return (package_name, PackageType.STANDARD) From 7e41f57336b9e5b4a54485f50c390badcebaa605 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 16:21:35 -0500 Subject: [PATCH 29/62] improve detection of bogus packages --- rope/contrib/autoimport/autoimport.py | 7 +++++-- rope/contrib/autoimport/utils.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 99f6c2781..ee01b30c8 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -268,7 +268,7 @@ def update_resource( resource_path, package_path, add_package_name=False ) package_tuple = get_package_name_from_path(package_path) - underlined = underlined if underlined else self.underlined + if package_tuple is None: return package_name = package_tuple[0] @@ -300,7 +300,7 @@ def _del_if_exist(self, module_name, commit: bool = True): self.connection.commit() def _get_avalible_packages( - self, + self, underlined: bool = False ) -> Tuple[List[pathlib.Path], List[str], List[Package]]: packages: List[pathlib.Path] = [] package_names: List[ @@ -310,6 +310,7 @@ def _get_avalible_packages( compiled_packages: List[str] = list(sys.builtin_module_names) folders = self.project.get_python_path_folders() existing = self._get_existing() + underlined = underlined if underlined else self.underlined for folder in folders: for package in pathlib.Path(folder.path).iterdir(): package_tuple = get_package_name_from_path(package) @@ -318,6 +319,8 @@ def _get_avalible_packages( package_name, package_type = package_tuple if package_name in existing: continue + if package_name.startswith("_") and not underlined: + continue if package_type == PackageType.COMPILED: compiled_packages.append(package_name) else: diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 669ab438e..d95819de4 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -19,6 +19,8 @@ def get_package_name_from_path( Returns None if not a viable package. """ package_name = package_path.name + if package_name.startswith(".") or package_name == "__pycache__": + return None if package_path.is_file(): if package_name.endswith(".so"): name = package_name.split(".")[0] From 2182bfe06296a03b953f7cb245452d583cc57875 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 5 Apr 2022 16:21:35 -0500 Subject: [PATCH 30/62] improve detection of bogus packages --- rope/contrib/autoimport/autoimport.py | 7 +++++-- rope/contrib/autoimport/utils.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 99f6c2781..8f0d42754 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -261,6 +261,7 @@ def update_resource( """Update the cache for global names in `resource`.""" resource_path: pathlib.Path = pathlib.Path(resource.real_path) package_path: pathlib.Path = self._project_path + underlined = underlined if underlined else self.underlined # The project doesn't need its name added to the path, # since the standard python file layout accounts for that # so we set add_package_name to False @@ -268,7 +269,6 @@ def update_resource( resource_path, package_path, add_package_name=False ) package_tuple = get_package_name_from_path(package_path) - underlined = underlined if underlined else self.underlined if package_tuple is None: return package_name = package_tuple[0] @@ -300,7 +300,7 @@ def _del_if_exist(self, module_name, commit: bool = True): self.connection.commit() def _get_avalible_packages( - self, + self, underlined: bool = False ) -> Tuple[List[pathlib.Path], List[str], List[Package]]: packages: List[pathlib.Path] = [] package_names: List[ @@ -310,6 +310,7 @@ def _get_avalible_packages( compiled_packages: List[str] = list(sys.builtin_module_names) folders = self.project.get_python_path_folders() existing = self._get_existing() + underlined = underlined if underlined else self.underlined for folder in folders: for package in pathlib.Path(folder.path).iterdir(): package_tuple = get_package_name_from_path(package) @@ -318,6 +319,8 @@ def _get_avalible_packages( package_name, package_type = package_tuple if package_name in existing: continue + if package_name.startswith("_") and not underlined: + continue if package_type == PackageType.COMPILED: compiled_packages.append(package_name) else: diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 669ab438e..d95819de4 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -19,6 +19,8 @@ def get_package_name_from_path( Returns None if not a viable package. """ package_name = package_path.name + if package_name.startswith(".") or package_name == "__pycache__": + return None if package_path.is_file(): if package_name.endswith(".so"): name = package_name.split(".")[0] From 24d0517b18cd227f3e126efb57928b195224f179 Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 6 Apr 2022 19:52:43 -0500 Subject: [PATCH 31/62] Move compiled modules to single thread, add single threaded option --- rope/contrib/autoimport/autoimport.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 8f0d42754..812aeb487 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -157,6 +157,7 @@ def generate_modules_cache( self, modules: List[str] = None, task_handle=taskhandle.NullTaskHandle(), + single_thread: bool = False, ): """ Generate global name cache for external modules listed in `modules`. @@ -187,12 +188,15 @@ def generate_modules_cache( packages.append(package_path) to_add.append((package_name,)) self._add_packages(to_add) - with ProcessPoolExecutor() as exectuor: - for name_list in exectuor.map(find_all_names_in_package, packages): - self._add_names(name_list) - for name_list in exectuor.map(get_names_from_compiled, compiled_packages): - self._add_names(name_list) - + if single_thread: + for package in packages: + self._add_names(find_all_names_in_package(package)) + else: + with ProcessPoolExecutor() as exectuor: + for name_list in exectuor.map(find_all_names_in_package, packages): + self._add_names(name_list) + for compiled_package in compiled_packages: + self._add_names(get_names_from_compiled(compiled_package)) self.connection.commit() def update_module(self, module: str): From fbc1cd318107f6f092085b27fbfdf0df6d033c6f Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 6 Apr 2022 20:25:00 -0500 Subject: [PATCH 32/62] propogate underlined, handle directories without __init__, detect compiled module's source properly --- rope/contrib/autoimport/autoimport.py | 50 ++++++++++++++++++++------- rope/contrib/autoimport/parse.py | 7 ++-- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 812aeb487..f51d06855 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -4,7 +4,7 @@ import sqlite3 import sys from concurrent.futures import ProcessPoolExecutor -from itertools import chain +from itertools import chain, repeat from typing import List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle @@ -16,7 +16,8 @@ from .parse import (find_all_names_in_package, get_names_from_compiled, get_names_from_file) from .utils import (get_modname_from_path, get_package_name_from_path, - sort_and_deduplicate, sort_and_deduplicate_tuple) + get_package_source, sort_and_deduplicate, + sort_and_deduplicate_tuple) class AutoImport: @@ -158,6 +159,7 @@ def generate_modules_cache( modules: List[str] = None, task_handle=taskhandle.NullTaskHandle(), single_thread: bool = False, + underlined: bool = False, ): """ Generate global name cache for external modules listed in `modules`. @@ -168,10 +170,14 @@ def generate_modules_cache( use generate_resource_cache for that instead. """ packages: List[pathlib.Path] = [] - compiled_packages: List[str] = [] + compiled_packages: List[Tuple[str, Source]] = [] to_add: List[Package] = [] + if self.underlined: + underlined = True if modules is None: - packages, compiled_packages, to_add = self._get_avalible_packages() + packages, compiled_packages, to_add = self._get_avalible_packages( + underlined + ) else: existing = self._get_existing() for modname in modules: @@ -179,10 +185,19 @@ def generate_modules_cache( if mod_tuple is None: continue package_path, package_name, package_type = mod_tuple + if package_name.startswith("_") and not underlined: + continue if package_name in existing: continue if package_type in (PackageType.COMPILED, PackageType.BUILTIN): - compiled_packages.append(package_name) + if package_type is PackageType.COMPILED: + assert ( + package_path is not None + ) # It should have been found, and isn't a builtin + source = get_package_source(package_path, self.project) + else: + source = Source.BUILTIN + compiled_packages.append((package_name, source)) else: assert package_path # Should only return none for a builtin packages.append(package_path) @@ -190,13 +205,20 @@ def generate_modules_cache( self._add_packages(to_add) if single_thread: for package in packages: - self._add_names(find_all_names_in_package(package)) + self._add_names( + find_all_names_in_package(package, underlined=underlined) + ) else: + underlined_list = repeat(underlined, len(packages)) with ProcessPoolExecutor() as exectuor: - for name_list in exectuor.map(find_all_names_in_package, packages): + for name_list in exectuor.map( + find_all_names_in_package, packages, underlined_list + ): self._add_names(name_list) - for compiled_package in compiled_packages: - self._add_names(get_names_from_compiled(compiled_package)) + for compiled_package, source in compiled_packages: + self._add_names( + get_names_from_compiled(compiled_package, source, underlined) + ) self.connection.commit() def update_module(self, module: str): @@ -305,13 +327,15 @@ def _del_if_exist(self, module_name, commit: bool = True): def _get_avalible_packages( self, underlined: bool = False - ) -> Tuple[List[pathlib.Path], List[str], List[Package]]: + ) -> Tuple[List[pathlib.Path], List[Tuple[str, Source]], List[Package]]: packages: List[pathlib.Path] = [] package_names: List[ Package ] = [] # List of packages to add to the package table # Get builtins first - compiled_packages: List[str] = list(sys.builtin_module_names) + compiled_packages: List[Tuple[str, Source]] = [ + (module, Source.BUILTIN) for module in sys.builtin_module_names + ] folders = self.project.get_python_path_folders() existing = self._get_existing() underlined = underlined if underlined else self.underlined @@ -326,7 +350,9 @@ def _get_avalible_packages( if package_name.startswith("_") and not underlined: continue if package_type == PackageType.COMPILED: - compiled_packages.append(package_name) + compiled_packages.append( + (package_name, get_package_source(package, self.project)) + ) else: packages.append(package) package_names.append((package_name,)) diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index c2892abd3..c532f5835 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -28,7 +28,7 @@ def get_names( """ if modpath.is_dir(): names: List[Name] - if modpath / "__init__.py": + if (modpath / "__init__.py").exists(): names = get_names_from_file( modpath / "__init__.py", modname, @@ -165,6 +165,7 @@ def find_all_names_in_package( def get_names_from_compiled( package: str, + source: Source, underlined: bool = False, ) -> List[Name]: """ @@ -188,7 +189,7 @@ def get_names_from_compiled( return [] if hasattr(module, "__all__"): for name in module.__all__: - results.append((str(name), package, package, Source.BUILTIN.value)) + results.append((str(name), package, package, source.value)) else: for name, value in inspect.getmembers(module): if underlined or not name.startswith("_"): @@ -197,5 +198,5 @@ def get_names_from_compiled( or inspect.isfunction(value) or inspect.isbuiltin(value) ): - results.append((str(name), package, package, Source.BUILTIN.value)) + results.append((str(name), package, package, source.value)) return results From ed5876a0e96933ed25c7247972ed10f91e1aa15f Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 6 Apr 2022 20:28:03 -0500 Subject: [PATCH 33/62] update docs on manual sources --- rope/contrib/autoimport/defs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index efa7d7c93..e888a918a 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -7,7 +7,7 @@ class Source(Enum): """Describes the source of the package, for sorting purposes.""" PROJECT = 0 # Obviously any project packages come first - MANUAL = 1 # Any packages manually added are probably important to the user + MANUAL = 1 # Placeholder since Autoimport classifies manually added modules BUILTIN = 2 STANDARD = 3 # We want to favor standard library items SITE_PACKAGE = 4 @@ -17,6 +17,7 @@ class Source(Enum): Name = Tuple[str, str, str, int] Package = Tuple[str] + class PackageType(Enum): """Describes the type of package, to determine how to get the names from it.""" From 056ec2c0153dd670eb7eb4c42be7ba54c8c7de12 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 7 Apr 2022 14:00:19 -0500 Subject: [PATCH 34/62] reformat with black --- rope/contrib/autoimport/autoimport.py | 17 ++++++++++++----- rope/contrib/autoimport/defs.py | 2 +- rope/contrib/autoimport/parse.py | 8 ++++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index f51d06855..d4228e611 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -13,11 +13,18 @@ from rope.refactor import importutils from .defs import Name, Package, PackageType, Source -from .parse import (find_all_names_in_package, get_names_from_compiled, - get_names_from_file) -from .utils import (get_modname_from_path, get_package_name_from_path, - get_package_source, sort_and_deduplicate, - sort_and_deduplicate_tuple) +from .parse import ( + find_all_names_in_package, + get_names_from_compiled, + get_names_from_file, +) +from .utils import ( + get_modname_from_path, + get_package_name_from_path, + get_package_source, + sort_and_deduplicate, + sort_and_deduplicate_tuple, +) class AutoImport: diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index e888a918a..5c55895a9 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -7,7 +7,7 @@ class Source(Enum): """Describes the source of the package, for sorting purposes.""" PROJECT = 0 # Obviously any project packages come first - MANUAL = 1 # Placeholder since Autoimport classifies manually added modules + MANUAL = 1 # Placeholder since Autoimport classifies manually added modules BUILTIN = 2 STANDARD = 3 # We want to favor standard library items SITE_PACKAGE = 4 diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index c532f5835..81799ecfc 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -11,8 +11,12 @@ from typing import List, Tuple from .defs import Name, PackageType, Source -from .utils import (get_modname_from_path, get_package_name_from_path, - get_package_source, submodules) +from .utils import ( + get_modname_from_path, + get_package_name_from_path, + get_package_source, + submodules, +) def get_names( From 2dfa97b7d6e68c922fbc171e3ab594f96e0e25db Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 7 Apr 2022 14:30:06 -0500 Subject: [PATCH 35/62] python 3.6 compatibility --- rope/contrib/autoimport/autoimport.py | 8 +++++--- rope/contrib/autoimport/parse.py | 2 +- rope/contrib/autoimport/utils.py | 20 +++++++++++--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index d4228e611..4d845ef4b 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -241,12 +241,14 @@ def close(self): def get_name_locations(self, name): """Return a list of ``(resource, lineno)`` tuples.""" result = [] - names = self.connection.execute( + modules = self.connection.execute( "select module from names where name like (?)", (name,) ).fetchall() - for module in names: + for module in modules: try: - module_name = module[0].removeprefix(f"{self._project_name}.") + module_name = module[0] + if module_name.startswith(f"{self._project_name}."): + module_name = ".".join(module_name.split(".")) pymodule = self.project.get_module(module_name) if name in pymodule: pyname = pymodule[name] diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 81799ecfc..71ef462be 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -47,7 +47,7 @@ def get_names( names.extend( get_names_from_file( file, - modname + f".{file.name.removesuffix('.py')}", + modname + f".{file.stem}", package_name, package_source, underlined=underlined, diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index d95819de4..c98561f68 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -26,7 +26,7 @@ def get_package_name_from_path( name = package_name.split(".")[0] return (name, PackageType.COMPILED) if package_name.endswith(".py"): - stripped_name = package_name.removesuffix(".py") + stripped_name = package_path.stem return (stripped_name, PackageType.SINGLE_FILE) return None if package_name.endswith((".egg-info", ".dist-info")): @@ -39,15 +39,17 @@ def get_modname_from_path( ) -> str: """Get module name from a path in respect to package.""" package_name: str = package_path.name - modname = ( - modpath.relative_to(package_path) - .as_posix() - .removesuffix("/__init__.py") - .removesuffix(".py") - .replace("/", ".") - ) + rel_path_parts = modpath.relative_to(package_path).parts + modname = "" + for part in rel_path_parts[:-1]: + modname += part + modname += "." + if rel_path_parts[-1] == "__init__": + modname = modname[:-1] + else: + modname = modname + modpath.stem if add_package_name: - modname = package_name if modname == "." else package_name + "." + modname + modname = package_name if modname == "" else package_name + "." + modname else: assert modname != "." return modname From 92dff7331b6e07132430d6308ca8caa5181d7407 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 7 Apr 2022 14:38:28 -0500 Subject: [PATCH 36/62] add some more tests --- rope/contrib/autoimport/utils.py | 15 ++++++++------- ropetest/contrib/autoimporttest.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index c98561f68..b23413c57 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -41,13 +41,14 @@ def get_modname_from_path( package_name: str = package_path.name rel_path_parts = modpath.relative_to(package_path).parts modname = "" - for part in rel_path_parts[:-1]: - modname += part - modname += "." - if rel_path_parts[-1] == "__init__": - modname = modname[:-1] - else: - modname = modname + modpath.stem + if len(rel_path_parts) > 0: + for part in rel_path_parts[:-1]: + modname += part + modname += "." + if rel_path_parts[-1] == "__init__": + modname = modname[:-1] + else: + modname = modname + modpath.stem if add_package_name: modname = package_name if modname == "" else package_name + "." + modname else: diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index dc183a1f4..301230055 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -121,6 +121,25 @@ def test_handling_builtin_modules(self): self.importer.update_module("sys") self.assertTrue("sys" in self.importer.get_modules("exit")) + def test_search(self): + self.importer.update_module("typing") + self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + + def test_generate_full_cache(self): + self.importer.generate_modules_cache() + self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + self.assertTrue(len(self.importer._dump_all()) > 0) + for table in self.importer._dump_all(): + self.assertTrue(len(table) > 0) + + def test_generate_full_cache_st(self): + """The single thread test takes much longer than the multithread test but is easier to debug""" + self.importer.generate_modules_cache(single_thread=True) + self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + self.assertTrue(len(self.importer._dump_all()) > 0) + for table in self.importer._dump_all(): + self.assertTrue(len(table) > 0) + class AutoImportObservingTest(unittest.TestCase): def setUp(self): From 1511451543e57708e5dfd5edc4f2251f7879e3ef Mon Sep 17 00:00:00 2001 From: bageljr Date: Fri, 8 Apr 2022 14:26:41 -0500 Subject: [PATCH 37/62] add utils tests --- rope/contrib/autoimport/utils.py | 2 +- ropetest/contrib/autoimport/conftest.py | 60 ++++++++++++++++++++++++ ropetest/contrib/autoimport/parsetest.py | 26 ++++++++++ ropetest/contrib/autoimport/utilstest.py | 58 +++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 ropetest/contrib/autoimport/conftest.py create mode 100644 ropetest/contrib/autoimport/parsetest.py create mode 100644 ropetest/contrib/autoimport/utilstest.py diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index b23413c57..fbf1060fb 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -38,7 +38,7 @@ def get_modname_from_path( modpath: pathlib.Path, package_path: pathlib.Path, add_package_name: bool = True ) -> str: """Get module name from a path in respect to package.""" - package_name: str = package_path.name + package_name: str = package_path.stem rel_path_parts = modpath.relative_to(package_path).parts modname = "" if len(rel_path_parts) > 0: diff --git a/ropetest/contrib/autoimport/conftest.py b/ropetest/contrib/autoimport/conftest.py new file mode 100644 index 000000000..7e7044032 --- /dev/null +++ b/ropetest/contrib/autoimport/conftest.py @@ -0,0 +1,60 @@ +import pathlib + +import pytest + +from ropetest import testutils + + +@pytest.fixture +def project(): + project = testutils.sample_project() + yield project + testutils.remove_project(project) + + +@pytest.fixture +def mod1(project): + mod1 = testutils.create_module(project, "mod1") + yield mod1 + + +@pytest.fixture +def mod1_path(mod1): + yield pathlib.Path(mod1.real_path) + + +@pytest.fixture +def project_path(project): + yield pathlib.Path(project.address) + + +@pytest.fixture +def typing_path(): + import typing + + yield pathlib.Path(typing.__file__) + + + + +@pytest.fixture +def build_env_path(): + from build import env + + yield pathlib.Path(env.__file__) + + +@pytest.fixture +def build_path(): + import build + + # Uses __init__.py so we need the parent + + yield pathlib.Path(build.__file__).parent + + +@pytest.fixture +def zlib_path(): + import zlib + + yield pathlib.Path(zlib.__file__) diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py new file mode 100644 index 000000000..87cc3186a --- /dev/null +++ b/ropetest/contrib/autoimport/parsetest.py @@ -0,0 +1,26 @@ +from itertools import chain + +from rope.contrib.autoimport import parse +from rope.contrib.autoimport.defs import PackageType, Source + + +def test_typing_names(typing_path): + names = parse.get_names_from_file( + typing_path, typing_path.stem, typing_path, Source.STANDARD + ) + assert "Dict" in chain(*names) + + +def test_get_typing_names(typing_path): + names = parse.get_names(typing_path, typing_path.stem, typing_path, Source.STANDARD) + assert "Dict" in chain(*names) + + +def test_find_all_typing_names(typing_path): + names = parse.find_all_names_in_package(typing_path) + assert "Dict" in chain(*names) + + +def test_find_sys(): + names = parse.get_names_from_compiled("sys", Source.BUILTIN) + assert "exit" in chain(*names) diff --git a/ropetest/contrib/autoimport/utilstest.py b/ropetest/contrib/autoimport/utilstest.py new file mode 100644 index 000000000..5b2d3e4df --- /dev/null +++ b/ropetest/contrib/autoimport/utilstest.py @@ -0,0 +1,58 @@ +"""Tests for autoimport utility functions, written in pytest""" +import pathlib + +from rope.contrib.autoimport import utils +from rope.contrib.autoimport.defs import PackageType, Source + + +def test_get_package_source(mod1_path, project): + assert utils.get_package_source(mod1_path, project) == Source.PROJECT + + +def test_get_package_source_not_project(mod1_path): + assert utils.get_package_source(mod1_path) == Source.UNKNOWN + + +def test_get_package_source_pytest(build_path): + # pytest is not installed as part of the standard library + # but should be installed into site_packages, + # so it should return Source.SITE_PACKAGE + assert utils.get_package_source(build_path) == Source.SITE_PACKAGE + + +def test_get_package_source_typing(typing_path): + + assert utils.get_package_source(typing_path) == Source.STANDARD + + +def test_get_modname_project_no_add(mod1_path, project_path): + + assert utils.get_modname_from_path(mod1_path, project_path, False) == "mod1" + + +def test_get_modname_single_file(typing_path): + + assert utils.get_modname_from_path(typing_path, typing_path) == "typing" + + +def test_get_modname_folder(build_path, build_env_path): + + assert utils.get_modname_from_path(build_env_path, build_path) == "build.env" + + +def test_get_package_name_sample(project_path): + package_name, package_type = utils.get_package_name_from_path(project_path) + assert package_name == "sample_project" + assert package_type == PackageType.STANDARD + + +def test_get_package_name_typing(typing_path): + package_name, package_type = utils.get_package_name_from_path(typing_path) + assert package_name == "typing" + assert package_type == PackageType.SINGLE_FILE + + +def test_get_package_name_compiled(zlib_path): + package_name, package_type = utils.get_package_name_from_path(zlib_path) + assert package_name == "zlib" + assert package_type == PackageType.COMPILED From 50d8f9e9883e6e01a04d937054a955cbb03b2165 Mon Sep 17 00:00:00 2001 From: bageljr Date: Sat, 9 Apr 2022 15:36:34 -0500 Subject: [PATCH 38/62] fix several issues with handling non-python files --- rope/contrib/autoimport/autoimport.py | 32 +++++++++++++-------------- rope/contrib/autoimport/parse.py | 16 ++++++-------- ropetest/contrib/autoimporttest.py | 14 ++++++------ 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 4d845ef4b..97cc41ac0 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -3,6 +3,7 @@ import re import sqlite3 import sys +from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from itertools import chain, repeat from typing import List, Optional, Tuple @@ -13,18 +14,11 @@ from rope.refactor import importutils from .defs import Name, Package, PackageType, Source -from .parse import ( - find_all_names_in_package, - get_names_from_compiled, - get_names_from_file, -) -from .utils import ( - get_modname_from_path, - get_package_name_from_path, - get_package_source, - sort_and_deduplicate, - sort_and_deduplicate_tuple, -) +from .parse import (find_all_names_in_package, get_names_from_compiled, + get_names_from_file) +from .utils import (get_modname_from_path, get_package_name_from_path, + get_package_source, sort_and_deduplicate, + sort_and_deduplicate_tuple) class AutoImport: @@ -334,6 +328,11 @@ def _del_if_exist(self, module_name, commit: bool = True): if commit: self.connection.commit() + def _get_python_folders(self) -> List[pathlib.Path]: + folders = self.project.get_python_path_folders() + folder_paths = [pathlib.Path(folder.path) for folder in folders if folder.path != "/usr/bin"] + return list(OrderedDict.fromkeys(folder_paths)) + def _get_avalible_packages( self, underlined: bool = False ) -> Tuple[List[pathlib.Path], List[Tuple[str, Source]], List[Package]]: @@ -345,11 +344,10 @@ def _get_avalible_packages( compiled_packages: List[Tuple[str, Source]] = [ (module, Source.BUILTIN) for module in sys.builtin_module_names ] - folders = self.project.get_python_path_folders() existing = self._get_existing() underlined = underlined if underlined else self.underlined - for folder in folders: - for package in pathlib.Path(folder.path).iterdir(): + for folder in self._get_python_folders(): + for package in folder.iterdir(): package_tuple = get_package_name_from_path(package) if package_tuple is None: continue @@ -431,8 +429,8 @@ def _find_package_path( ) -> Optional[Tuple[Optional[pathlib.Path], str, PackageType]]: if target_name in sys.builtin_module_names: return (None, target_name, PackageType.BUILTIN) - for folder in self.project.get_python_path_folders(): - for package in pathlib.Path(folder.path).iterdir(): + for folder in self._get_python_folders(): + for package in folder.iterdir(): package_tuple = get_package_name_from_path(package) if package_tuple is None: continue diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 71ef462be..95254bf3f 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -11,12 +11,8 @@ from typing import List, Tuple from .defs import Name, PackageType, Source -from .utils import ( - get_modname_from_path, - get_package_name_from_path, - get_package_source, - submodules, -) +from .utils import (get_modname_from_path, get_package_name_from_path, + get_package_source, submodules) def get_names( @@ -54,9 +50,11 @@ def get_names( ) ) return names - return get_names_from_file( - modpath, modname, package_name, package_source, underlined=underlined - ) + if modpath.suffix == ".py": + return get_names_from_file( + modpath, modname, package_name, package_source, underlined=underlined + ) + return [] def parse_all(node: ast.Assign, modname: str, package: str, package_source: Source): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 301230055..2bf60b189 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -132,13 +132,13 @@ def test_generate_full_cache(self): for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) - def test_generate_full_cache_st(self): - """The single thread test takes much longer than the multithread test but is easier to debug""" - self.importer.generate_modules_cache(single_thread=True) - self.assertTrue("from typing import Dict" in self.importer.search("Dict")) - self.assertTrue(len(self.importer._dump_all()) > 0) - for table in self.importer._dump_all(): - self.assertTrue(len(table) > 0) + # def test_generate_full_cache_st(self): + # """The single thread test takes much longer than the multithread test but is easier to debug""" + # self.importer.generate_modules_cache(single_thread=True) + # self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + # self.assertTrue(len(self.importer._dump_all()) > 0) + # for table in self.importer._dump_all(): + # self.assertTrue(len(table) > 0) class AutoImportObservingTest(unittest.TestCase): From 5ef7608b8e74be8ced9a0b75accf302b6bd089f7 Mon Sep 17 00:00:00 2001 From: bageljr Date: Sat, 9 Apr 2022 15:58:13 -0500 Subject: [PATCH 39/62] add exact_match toggle --- rope/contrib/autoimport/autoimport.py | 26 +++++++++++++++++--------- ropetest/contrib/autoimporttest.py | 23 ++++++++++++++++++++++- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 97cc41ac0..6e16e0492 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -89,23 +89,29 @@ def import_assist(self, starting: str): results ) # Remove duplicates from multiple occurences of the same item - def search(self, name) -> List[str]: + def search(self, name: str, exact_match: bool = False) -> List[str]: """Search both modules and names for an import string.""" + if not exact_match: + name = name + "%" # Makes the query a starts_with query results: List[Tuple[str, int]] = [] - for module, source in self.connection.execute( - "SELECT module, source FROM names WHERE name LIKE (?)", (name,) + for import_name, module, source in self.connection.execute( + "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) ): - results.append((f"from {module} import {name}", source)) + results.append((f"from {module} import {import_name}", source)) for module, source in self.connection.execute( "Select module, source FROM names where module LIKE (?)", ("%." + name,) ): - results.append( - (f"from {module.removesuffix(f'.{name}')} import {name}", source) - ) + parts = module.split(".") + import_name = parts[-1] + remaining = parts[0] + for part in parts[1:-1]: + remaining += "." + remaining += part + results.append((f"from {remaining} import {import_name}", source)) for module, source in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): - results.append((f"import {name}", source)) + results.append((f"import {module}", source)) return sort_and_deduplicate(results) def get_modules(self, name) -> List[str]: @@ -330,7 +336,9 @@ def _del_if_exist(self, module_name, commit: bool = True): def _get_python_folders(self) -> List[pathlib.Path]: folders = self.project.get_python_path_folders() - folder_paths = [pathlib.Path(folder.path) for folder in folders if folder.path != "/usr/bin"] + folder_paths = [ + pathlib.Path(folder.path) for folder in folders if folder.path != "/usr/bin" + ] return list(OrderedDict.fromkeys(folder_paths)) def _get_avalible_packages( diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 2bf60b189..1adaded66 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -121,9 +121,30 @@ def test_handling_builtin_modules(self): self.importer.update_module("sys") self.assertTrue("sys" in self.importer.get_modules("exit")) + def test_search_submodule(self): + self.importer.update_module("os") + self.assertTrue( + "from os import path" in self.importer.search("path", exact_match=True) + ) + self.assertTrue("from os import path" in self.importer.search("pa")) + self.assertTrue("from os import path" in self.importer.search("path")) + + def test_search_module(self): + self.importer.update_module("os") + self.assertTrue("import os" in self.importer.search("os", exact_match=True)) + self.assertTrue("import os" in self.importer.search("os")) + self.assertTrue("import os" in self.importer.search("o")) + def test_search(self): self.importer.update_module("typing") - self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + import_statement = "from typing import Dict" + self.assertTrue( + import_statement in self.importer.search("Dict", exact_match=True) + ) + self.assertTrue(import_statement in self.importer.search("Dict")) + self.assertTrue(import_statement in self.importer.search("Dic")) + self.assertTrue(import_statement in self.importer.search("Di")) + self.assertTrue(import_statement in self.importer.search("D")) def test_generate_full_cache(self): self.importer.generate_modules_cache() From c3eaf26a3771e961a208a35e3eb6a5f52ead9fe3 Mon Sep 17 00:00:00 2001 From: bageljr Date: Sun, 10 Apr 2022 20:52:06 -0500 Subject: [PATCH 40/62] handle python_crun, return module name with search --- rope/contrib/autoimport/autoimport.py | 58 +++++++++++++++++------- rope/contrib/autoimport/parse.py | 13 ++++-- ropetest/contrib/autoimport/parsetest.py | 3 ++ ropetest/contrib/autoimporttest.py | 30 ++++++------ 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 6e16e0492..c7c39fcef 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -1,4 +1,5 @@ """AutoImport module for rope.""" +import logging import pathlib import re import sqlite3 @@ -11,14 +12,22 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource +from rope.contrib.autoimport.defs import Name, Package, PackageType, Source +from rope.contrib.autoimport.parse import ( + find_all_names_in_package, + get_names_from_compiled, + get_names_from_file, +) +from rope.contrib.autoimport.utils import ( + get_modname_from_path, + get_package_name_from_path, + get_package_source, + sort_and_deduplicate, + sort_and_deduplicate_tuple, +) from rope.refactor import importutils -from .defs import Name, Package, PackageType, Source -from .parse import (find_all_names_in_package, get_names_from_compiled, - get_names_from_file) -from .utils import (get_modname_from_path, get_package_name_from_path, - get_package_source, sort_and_deduplicate, - sort_and_deduplicate_tuple) +logger = logging.getLogger(__name__) class AutoImport: @@ -49,7 +58,11 @@ def __init__(self, project, observe=True, underlined=False, memory=True): """ self.project = project self.underlined = underlined - db_path = ":memory:" if memory else f"{project.ropefolder.path}/autoimport.db" + db_path: str + if memory or project.ropefolder is None: + db_path = ":memory:" + else: + db_path = f"{project.ropefolder.path}/autoimport.db" self.connection = sqlite3.connect(db_path) self._setup_db() self._check_all() @@ -89,15 +102,19 @@ def import_assist(self, starting: str): results ) # Remove duplicates from multiple occurences of the same item - def search(self, name: str, exact_match: bool = False) -> List[str]: - """Search both modules and names for an import string.""" + def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: + """ + Search both modules and names for an import string. + + Returns list of import statement ,modname pairs + """ if not exact_match: name = name + "%" # Makes the query a starts_with query - results: List[Tuple[str, int]] = [] + results: List[Tuple[str, str, int]] = [] for import_name, module, source in self.connection.execute( "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) ): - results.append((f"from {module} import {import_name}", source)) + results.append((f"from {module} import {import_name}", import_name, source)) for module, source in self.connection.execute( "Select module, source FROM names where module LIKE (?)", ("%." + name,) ): @@ -107,12 +124,14 @@ def search(self, name: str, exact_match: bool = False) -> List[str]: for part in parts[1:-1]: remaining += "." remaining += part - results.append((f"from {remaining} import {import_name}", source)) + results.append( + (f"from {remaining} import {import_name}", import_name, source) + ) for module, source in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): - results.append((f"import {module}", source)) - return sort_and_deduplicate(results) + results.append((f"import {module}", module, source)) + return sort_and_deduplicate_tuple(results) def get_modules(self, name) -> List[str]: """Get the list of modules that have global `name`.""" @@ -223,9 +242,14 @@ def generate_modules_cache( ): self._add_names(name_list) for compiled_package, source in compiled_packages: - self._add_names( - get_names_from_compiled(compiled_package, source, underlined) - ) + try: + self._add_names( + get_names_from_compiled(compiled_package, source, underlined) + ) + except Exception as e: + logger.error( + f"{compiled_package} could not be imported for autoimport analysis" + ) self.connection.commit() def update_module(self, module: str): diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 95254bf3f..0d4a9aa7f 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -11,8 +11,12 @@ from typing import List, Tuple from .defs import Name, PackageType, Source -from .utils import (get_modname_from_path, get_package_name_from_path, - get_package_source, submodules) +from .utils import ( + get_modname_from_path, + get_package_name_from_path, + get_package_source, + submodules, +) def get_names( @@ -181,7 +185,10 @@ def get_names_from_compiled( underlined : bool include underlined names """ - if package == "builtins" or (package.startswith("_") and not underlined): + # builtins is banned because you never have to import it + # python_crun is banned because it crashes python + banned = ["builtins", "python_crun"] + if package in banned or (package.startswith("_") and not underlined): return [] # Builtins is redundant since you don't have to import it. results: List[Name] = [] try: diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index 87cc3186a..aba4c8ab5 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -24,3 +24,6 @@ def test_find_all_typing_names(typing_path): def test_find_sys(): names = parse.get_names_from_compiled("sys", Source.BUILTIN) assert "exit" in chain(*names) +def test_find_underlined(): + names = parse.get_names_from_compiled("os", Source.BUILTIN, underlined=True) + assert "_exit" in chain(*names) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 1adaded66..400ff00aa 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -28,12 +28,6 @@ def test_update_resource(self): self.importer.update_resource(self.mod1) self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) - # def test_update_module(self): - # update module only works on external modules. - # self.mod1.write("myvar = None") - # self.importer.update_module("rope") - # self.assertEqual([("myvar", "mod1")], self.importer.import_assist("myva")) - def test_update_non_existent_module(self): self.importer.update_module("does_not_exists_this") self.assertEqual([], self.importer.import_assist("myva")) @@ -123,21 +117,25 @@ def test_handling_builtin_modules(self): def test_search_submodule(self): self.importer.update_module("os") + import_statement = ("from os import path", "path") self.assertTrue( - "from os import path" in self.importer.search("path", exact_match=True) + import_statement in self.importer.search("path", exact_match=True) ) - self.assertTrue("from os import path" in self.importer.search("pa")) - self.assertTrue("from os import path" in self.importer.search("path")) + self.assertTrue(import_statement in self.importer.search("pa")) + self.assertTrue(import_statement in self.importer.search("path")) def test_search_module(self): self.importer.update_module("os") - self.assertTrue("import os" in self.importer.search("os", exact_match=True)) - self.assertTrue("import os" in self.importer.search("os")) - self.assertTrue("import os" in self.importer.search("o")) + import_statement = ("import os", "os") + self.assertTrue( + import_statement in self.importer.search("os", exact_match=True) + ) + self.assertTrue(import_statement in self.importer.search("os")) + self.assertTrue(import_statement in self.importer.search("o")) def test_search(self): self.importer.update_module("typing") - import_statement = "from typing import Dict" + import_statement = ("from typing import Dict", "Dict") self.assertTrue( import_statement in self.importer.search("Dict", exact_match=True) ) @@ -148,7 +146,9 @@ def test_search(self): def test_generate_full_cache(self): self.importer.generate_modules_cache() - self.assertTrue("from typing import Dict" in self.importer.search("Dict")) + self.assertTrue( + ("from typing import Dict", "Dict") in self.importer.search("Dict") + ) self.assertTrue(len(self.importer._dump_all()) > 0) for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) @@ -188,3 +188,5 @@ def test_removing_files(self): self.mod1.write("myvar = None\n") self.mod1.remove() self.assertEqual([], self.importer.get_modules("myvar")) + + From 2a7077fbf8e3158d07851b868ad1fd4716ddba31 Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 11 Apr 2022 19:12:14 -0500 Subject: [PATCH 41/62] add lsp search --- rope/contrib/autoimport/autoimport.py | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index c7c39fcef..29533b564 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -133,6 +133,42 @@ def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: results.append((f"import {module}", module, source)) return sort_and_deduplicate_tuple(results) + def lsp_search( + self, name: str, exact_match: bool = False + ) -> Tuple[List[Tuple[str, str, int]], List[Tuple[str, str, int]]]: + """ + Search both modules and names for an import string. + + Returns the name, import statement, source, split into normal names and modules. + """ + if not exact_match: + name = name + "%" # Makes the query a starts_with query + results_name: List[Tuple[str, str, int]] = [] + results_module: List[Tuple[str, str, int]] = [] + for import_name, module, source in self.connection.execute( + "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) + ): + results_name.append( + (f"from {module} import {import_name}", import_name, source) + ) + for module, source in self.connection.execute( + "Select module, source FROM names where module LIKE (?)", ("%." + name,) + ): + parts = module.split(".") + import_name = parts[-1] + remaining = parts[0] + for part in parts[1:-1]: + remaining += "." + remaining += part + results_module.append( + (f"from {remaining} import {import_name}", import_name, source) + ) + for module, source in self.connection.execute( + "Select module, source from names where module LIKE (?)", (name,) + ): + results_module.append((f"import {module}", module, source)) + return results_name, results_module + def get_modules(self, name) -> List[str]: """Get the list of modules that have global `name`.""" results = self.connection.execute( From 46252f69323042a5926b7be95033c20f1d37933b Mon Sep 17 00:00:00 2001 From: bageljr Date: Mon, 11 Apr 2022 20:10:45 -0500 Subject: [PATCH 42/62] add nametype --- rope/contrib/autoimport/autoimport.py | 2 +- rope/contrib/autoimport/defs.py | 30 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 6e16e0492..90902854d 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -61,7 +61,7 @@ def __init__(self, project, observe=True, underlined=False, memory=True): def _setup_db(self): packages_table = "(pacakge TEXT)" - names_table = "(name TEXT, module TEXT, package TEXT, source INTEGER)" + names_table = "(name TEXT, module TEXT, package TEXT, source INTEGER, type INTEGER)" self.connection.execute(f"create table if not exists names{names_table}") self.connection.execute(f"create table if not exists packages{packages_table}") self.connection.commit() diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 5c55895a9..bfe4226c5 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -25,3 +25,33 @@ class PackageType(Enum): STANDARD = 1 # Just a folder COMPILED = 2 # .so module SINGLE_FILE = 3 # a .py file + + +class NameType(Enum): + """Describes the type of Name for lsp completions. Taken from python lsp server""" + + Text = 1 + Method = 2 + Function = 3 + Constructor = 4 + Field = 5 + Variable = 6 + Class = 7 + Interface = 8 + Module = 9 + Property = 10 + Unit = 11 + Value = 12 + Enum = 13 + Keyword = 14 + Snippet = 15 + Color = 16 + File = 17 + Reference = 18 + Folder = 19 + EnumMember = 20 + Constant = 21 + Struct = 22 + Event = 23 + Operator = 24 + TypeParameter = 25 From b793a853d1df8365455539c3f4f20cbfb6b77427 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 12 Apr 2022 16:58:03 -0500 Subject: [PATCH 43/62] Update workflows to install rope in editable mode --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f03fea3c6..0e9ba5c7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip pytest pytest-timeout + python -m pip -e .[dev] - name: Test with pytest run: | pytest -v From 8fe2817b93266f23c31c567055cfa5527c4c9553 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 12 Apr 2022 19:15:43 -0500 Subject: [PATCH 44/62] WIP use generators --- rope/contrib/autoimport/autoimport.py | 53 ++++++----- rope/contrib/autoimport/defs.py | 19 +++- rope/contrib/autoimport/parse.py | 114 ++++++++++++++--------- ropetest/contrib/autoimport/parsetest.py | 9 +- 4 files changed, 120 insertions(+), 75 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 18c1a069f..cf1e0239c 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -7,24 +7,20 @@ from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from itertools import chain, repeat -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Iterable from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource from rope.contrib.autoimport.defs import Name, Package, PackageType, Source -from rope.contrib.autoimport.parse import ( - find_all_names_in_package, - get_names_from_compiled, - get_names_from_file, -) -from rope.contrib.autoimport.utils import ( - get_modname_from_path, - get_package_name_from_path, - get_package_source, - sort_and_deduplicate, - sort_and_deduplicate_tuple, -) +from rope.contrib.autoimport.parse import (find_all_names_in_package, + get_names_from_compiled, + get_names_from_file) +from rope.contrib.autoimport.utils import (get_modname_from_path, + get_package_name_from_path, + get_package_source, + sort_and_deduplicate, + sort_and_deduplicate_tuple) from rope.refactor import importutils logger = logging.getLogger(__name__) @@ -74,7 +70,9 @@ def __init__(self, project, observe=True, underlined=False, memory=True): def _setup_db(self): packages_table = "(pacakge TEXT)" - names_table = "(name TEXT, module TEXT, package TEXT, source INTEGER, type INTEGER)" + names_table = ( + "(name TEXT, module TEXT, package TEXT, source INTEGER, type INTEGER)" + ) self.connection.execute(f"create table if not exists names{names_table}") self.connection.execute(f"create table if not exists packages{packages_table}") self.connection.commit() @@ -135,7 +133,7 @@ def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: def lsp_search( self, name: str, exact_match: bool = False - ) -> Tuple[List[Tuple[str, str, int]], List[Tuple[str, str, int]]]: + ) -> Tuple[List[Tuple[str, str, int, int]], List[Tuple[str, str, int, int]]]: """ Search both modules and names for an import string. @@ -143,16 +141,17 @@ def lsp_search( """ if not exact_match: name = name + "%" # Makes the query a starts_with query - results_name: List[Tuple[str, str, int]] = [] - results_module: List[Tuple[str, str, int]] = [] - for import_name, module, source in self.connection.execute( - "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) + results_name: List[Tuple[str, str, int, int]] = [] + results_module: List[Tuple[str, str, int, int]] = [] + for import_name, module, source, type in self.connection.execute( + "SELECT name, module, source, type FROM names WHERE name LIKE (?)", (name,) ): results_name.append( - (f"from {module} import {import_name}", import_name, source) + (f"from {module} import {import_name}", import_name, source, type) ) - for module, source in self.connection.execute( - "Select module, source FROM names where module LIKE (?)", ("%." + name,) + for module, source, type in self.connection.execute( + "Select module, source, type FROM names where module LIKE (?)", + ("%." + name,), ): parts = module.split(".") import_name = parts[-1] @@ -161,12 +160,12 @@ def lsp_search( remaining += "." remaining += part results_module.append( - (f"from {remaining} import {import_name}", import_name, source) + (f"from {remaining} import {import_name}", import_name, source, type) ) - for module, source in self.connection.execute( + for module, source, type in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): - results_module.append((f"import {module}", module, source)) + results_module.append((f"import {module}", module, source, type)) return results_name, results_module def get_modules(self, name) -> List[str]: @@ -468,9 +467,9 @@ def _removed(self, resource): modname = self._modname(resource) self._del_if_exist(modname) - def _add_names(self, names: List[Name]): + def _add_names(self, names: Iterable[Name]): self.connection.executemany( - "insert into names(name,module,package,source) values (?,?,?,?)", + "insert into names(name,module,package,source) values (?,?,?,?,?)", names, ) diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index bfe4226c5..67dc69e23 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -1,6 +1,7 @@ """Definitions of types for the Autoimport program.""" +import pathlib from enum import Enum -from typing import Tuple +from typing import NamedTuple, Tuple class Source(Enum): @@ -14,10 +15,16 @@ class Source(Enum): UNKNOWN = 5 -Name = Tuple[str, str, str, int] Package = Tuple[str] +class NameFile(NamedTuple): + """Descriptor of information to get names from a file using ast.""" + + filepath: pathlib.Path + modname: str + underlined: bool + class PackageType(Enum): """Describes the type of package, to determine how to get the names from it.""" @@ -55,3 +62,11 @@ class NameType(Enum): Event = 23 Operator = 24 TypeParameter = 25 + + +class Name(NamedTuple): + name: str + modname: str + package: str + source: Source + name_type: NameType diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 0d4a9aa7f..c1f008e10 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -8,15 +8,11 @@ import inspect import pathlib from importlib import import_module -from typing import List, Tuple +from typing import Generator, List, Optional, Tuple -from .defs import Name, PackageType, Source -from .utils import ( - get_modname_from_path, - get_package_name_from_path, - get_package_source, - submodules, -) +from .defs import Name, NameType, PackageType, Source +from .utils import (get_modname_from_path, get_package_name_from_path, + get_package_source, submodules) def get_names( @@ -25,7 +21,7 @@ def get_names( package_name: str, package_source: Source, underlined: bool = False, -) -> List[Name]: +) -> Generator[Name, None, None]: """Get all names in the `modname` module, located at modpath. `modname` is the name of a module. @@ -58,25 +54,46 @@ def get_names( return get_names_from_file( modpath, modname, package_name, package_source, underlined=underlined ) - return [] -def parse_all(node: ast.Assign, modname: str, package: str, package_source: Source): +def parse_all( + node: ast.Assign, modname: str, package: str, package_source: Source +) -> Generator[Name, None, None]: """Parse the node which contains the value __all__ and return its contents.""" # I assume that the __all__ value isn't assigned via tuple - all_results: List[Name] = [] assert isinstance(node.value, ast.List) for item in node.value.elts: assert isinstance(item, ast.Constant) - all_results.append( - ( - str(item.value), - modname, - package, - package_source.value, - ) - ) - return all_results + name_type: NameType = NameType.Keyword + # TODO somehow determine the actual value of this since every member of all is a string + yield Name(str(item.value), modname, package, package_source, name_type) + + +def get_type_ast(node: ast.AST) -> NameType: + """Get the lsp type of a node.""" + if isinstance(node, ast.ClassDef): + return NameType.Class + if isinstance(node, ast.FunctionDef): + return NameType.Function + if isinstance(node, ast.Assign): + return NameType.Variable + return NameType.Text # default value + + +def find_all(root_node: ast.AST) -> Optional[List[str]]: + """Find the contents of __all__.""" + for node in ast.iter_child_nodes(root_node): + if isinstance(node, ast.Assign): + for target in node.targets: + try: + assert isinstance(target, ast.Name) + if target.id == "__all__": + assert isinstance(node.value, ast.List) + return node.value.elts + except (AttributeError, AssertionError): + # TODO handle tuple assignment + pass + return None def get_names_from_file( @@ -86,7 +103,7 @@ def get_names_from_file( package_source: Source, only_all: bool = False, underlined: bool = False, -) -> List[Name]: +) -> Generator[Name, None, None]: """ Get all the names from a given file using ast. @@ -100,28 +117,33 @@ def get_names_from_file( root_node = ast.parse(file.read()) except SyntaxError as error: print(error) - return [] - results: List[Name] = [] + return + all = find_all(root_node) for node in ast.iter_child_nodes(root_node): - node_names: List[str] = [] if isinstance(node, ast.Assign): for target in node.targets: try: assert isinstance(target, ast.Name) - if target.id == "__all__": - return parse_all(node, modname, package, package_source) - node_names.append(target.id) + if underlined or not target.id.startswith("_"): + yield Name( + target.id, + modname, + package, + package_source, + get_type_ast(node), + ) except (AttributeError, AssertionError): # TODO handle tuple assignment pass elif isinstance(node, (ast.FunctionDef, ast.ClassDef)): - node_names = [node.name] - for node_name in node_names: - if underlined or not node_name.startswith("_"): - results.append((node_name, modname, package, package_source.value)) - if only_all: - return [] - return results + if underlined or not node.name.startswith("_"): + yield Name( + node.name, + modname, + package, + package_source, + get_type_ast(node), + ) def find_all_names_in_package( @@ -169,11 +191,19 @@ def find_all_names_in_package( return result +def get_type_object(object) -> NameType: + if inspect.isclass(object): + return NameType.Class + if inspect.ismethod(object): + return NameType.Function + return NameType.Constant + + def get_names_from_compiled( package: str, source: Source, underlined: bool = False, -) -> List[Name]: +) -> Generator[Name, None, None]: """ Get the names from a compiled module. @@ -189,16 +219,12 @@ def get_names_from_compiled( # python_crun is banned because it crashes python banned = ["builtins", "python_crun"] if package in banned or (package.startswith("_") and not underlined): - return [] # Builtins is redundant since you don't have to import it. - results: List[Name] = [] + return # Builtins is redundant since you don't have to import it. try: module = import_module(str(package)) except ImportError: # print(f"couldn't import {package}") - return [] - if hasattr(module, "__all__"): - for name in module.__all__: - results.append((str(name), package, package, source.value)) + return else: for name, value in inspect.getmembers(module): if underlined or not name.startswith("_"): @@ -207,5 +233,5 @@ def get_names_from_compiled( or inspect.isfunction(value) or inspect.isbuiltin(value) ): - results.append((str(name), package, package, source.value)) - return results + yield Name(str(name), package, package, source, get_type_object(name)) + diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index aba4c8ab5..cb2075a62 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -5,9 +5,12 @@ def test_typing_names(typing_path): - names = parse.get_names_from_file( - typing_path, typing_path.stem, typing_path, Source.STANDARD + names = list( + parse.get_names_from_file( + typing_path, typing_path.stem, typing_path, Source.STANDARD + ) ) + print(names) assert "Dict" in chain(*names) @@ -24,6 +27,8 @@ def test_find_all_typing_names(typing_path): def test_find_sys(): names = parse.get_names_from_compiled("sys", Source.BUILTIN) assert "exit" in chain(*names) + + def test_find_underlined(): names = parse.get_names_from_compiled("os", Source.BUILTIN, underlined=True) assert "_exit" in chain(*names) From 9e6d95618fa8530e9550c3cad8c0b429a9ee9937 Mon Sep 17 00:00:00 2001 From: bageljr Date: Tue, 12 Apr 2022 23:14:35 -0500 Subject: [PATCH 45/62] itemkind infrence for compiled modules --- rope/contrib/autoimport/defs.py | 7 +++++-- rope/contrib/autoimport/parse.py | 7 ++++--- ropetest/contrib/autoimport/parsetest.py | 23 +++++++++++++++-------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 67dc69e23..c4c35c718 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -1,7 +1,7 @@ """Definitions of types for the Autoimport program.""" import pathlib from enum import Enum -from typing import NamedTuple, Tuple +from typing import NamedTuple class Source(Enum): @@ -15,7 +15,9 @@ class Source(Enum): UNKNOWN = 5 -Package = Tuple[str] +class Package(NamedTuple): + name: str + # modified_time class NameFile(NamedTuple): @@ -25,6 +27,7 @@ class NameFile(NamedTuple): modname: str underlined: bool + class PackageType(Enum): """Describes the type of package, to determine how to get the names from it.""" diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index c1f008e10..bc5578366 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -194,7 +194,7 @@ def find_all_names_in_package( def get_type_object(object) -> NameType: if inspect.isclass(object): return NameType.Class - if inspect.ismethod(object): + if inspect.isfunction(object) or inspect.isbuiltin(object): return NameType.Function return NameType.Constant @@ -233,5 +233,6 @@ def get_names_from_compiled( or inspect.isfunction(value) or inspect.isbuiltin(value) ): - yield Name(str(name), package, package, source, get_type_object(name)) - + yield Name( + str(name), package, package, source, get_type_object(value) + ) diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index cb2075a62..b6e9fcb0d 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -1,7 +1,8 @@ from itertools import chain +from typing import Dict from rope.contrib.autoimport import parse -from rope.contrib.autoimport.defs import PackageType, Source +from rope.contrib.autoimport.defs import Name, NameType, PackageType, Source def test_typing_names(typing_path): @@ -11,24 +12,30 @@ def test_typing_names(typing_path): ) ) print(names) - assert "Dict" in chain(*names) + assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( + names + ) def test_get_typing_names(typing_path): names = parse.get_names(typing_path, typing_path.stem, typing_path, Source.STANDARD) - assert "Dict" in chain(*names) + assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( + names + ) def test_find_all_typing_names(typing_path): names = parse.find_all_names_in_package(typing_path) - assert "Dict" in chain(*names) + assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( + names + ) def test_find_sys(): - names = parse.get_names_from_compiled("sys", Source.BUILTIN) - assert "exit" in chain(*names) + names = list(parse.get_names_from_compiled("sys", Source.BUILTIN)) + assert Name("exit", "sys", "sys", Source.BUILTIN, NameType.Function) in names def test_find_underlined(): - names = parse.get_names_from_compiled("os", Source.BUILTIN, underlined=True) - assert "_exit" in chain(*names) + names = list(parse.get_names_from_compiled("os", Source.BUILTIN, underlined=True)) + assert Name("_exit", "os", "os", Source.BUILTIN, NameType.Function) in names From 277edf1a83121bf2ba5add0a6de5fcceabe90cbe Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 00:27:46 -0500 Subject: [PATCH 46/62] add more tuples and use them more --- rope/contrib/autoimport/autoimport.py | 3 +- rope/contrib/autoimport/defs.py | 20 ++- rope/contrib/autoimport/parse.py | 157 +++++++---------------- rope/contrib/autoimport/utils.py | 70 +++++++--- ropetest/contrib/autoimport/parsetest.py | 23 +--- ropetest/contrib/autoimport/utilstest.py | 27 ++-- 6 files changed, 131 insertions(+), 169 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index cf1e0239c..df0637562 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -13,11 +13,10 @@ from rope.base.project import Project from rope.base.resources import Resource from rope.contrib.autoimport.defs import Name, Package, PackageType, Source -from rope.contrib.autoimport.parse import (find_all_names_in_package, +from rope.contrib.autoimport.parse import ( get_names_from_compiled, get_names_from_file) from rope.contrib.autoimport.utils import (get_modname_from_path, - get_package_name_from_path, get_package_source, sort_and_deduplicate, sort_and_deduplicate_tuple) diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index c4c35c718..440ae4d44 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -14,9 +14,6 @@ class Source(Enum): SITE_PACKAGE = 4 UNKNOWN = 5 - -class Package(NamedTuple): - name: str # modified_time @@ -26,6 +23,7 @@ class NameFile(NamedTuple): filepath: pathlib.Path modname: str underlined: bool + process_imports: bool = False class PackageType(Enum): @@ -67,9 +65,25 @@ class NameType(Enum): TypeParameter = 25 +class Package(NamedTuple): + name: str + source: Source + path: pathlib.Path + type: PackageType + + class Name(NamedTuple): + """A Name to be added to the database""" + name: str modname: str package: str source: Source name_type: NameType + + +class PartialName(NamedTuple): + """Partial information of a Name""" + + name: str + name_type: NameType diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index bc5578366..6f2564f60 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -8,52 +8,49 @@ import inspect import pathlib from importlib import import_module -from typing import Generator, List, Optional, Tuple - -from .defs import Name, NameType, PackageType, Source -from .utils import (get_modname_from_path, get_package_name_from_path, - get_package_source, submodules) - - -def get_names( - modpath: pathlib.Path, - modname: str, - package_name: str, - package_source: Source, - underlined: bool = False, -) -> Generator[Name, None, None]: - """Get all names in the `modname` module, located at modpath. - - `modname` is the name of a module. - """ - if modpath.is_dir(): - names: List[Name] - if (modpath / "__init__.py").exists(): - names = get_names_from_file( - modpath / "__init__.py", - modname, - package_name, - package_source, - only_all=True, - ) - if len(names) > 0: - return names - names = [] - for file in modpath.glob("*.py"): - names.extend( - get_names_from_file( - file, - modname + f".{file.stem}", - package_name, - package_source, - underlined=underlined, - ) - ) - return names - if modpath.suffix == ".py": - return get_names_from_file( - modpath, modname, package_name, package_source, underlined=underlined - ) +from typing import Generator, List, Optional + +from .defs import Name, NameType, PartialName, Source + +# def get_names( +# modpath: pathlib.Path, +# modname: str, +# package_name: str, +# package_source: Source, +# underlined: bool = False, +# ) -> Generator[Name, None, None]: +# """Get all names in the `modname` module, located at modpath. +# +# `modname` is the name of a module. +# """ +# if modpath.is_dir(): +# names: List[Name] +# if (modpath / "__init__.py").exists(): +# names = get_names_from_file( +# modpath / "__init__.py", +# modname, +# package_name, +# package_source, +# only_all=True, +# ) +# if len(names) > 0: +# return names +# names = [] +# for file in modpath.glob("*.py"): +# names.extend( +# get_names_from_file( +# file, +# modname + f".{file.stem}", +# package_name, +# package_source, +# underlined=underlined, +# ) +# ) +# return names +# if modpath.suffix == ".py": +# return get_names_from_file( +# modpath, modname, package_name, package_source, underlined=underlined +# ) def parse_all( @@ -89,6 +86,7 @@ def find_all(root_node: ast.AST) -> Optional[List[str]]: assert isinstance(target, ast.Name) if target.id == "__all__": assert isinstance(node.value, ast.List) + assert node.value.elts is not None return node.value.elts except (AttributeError, AssertionError): # TODO handle tuple assignment @@ -98,19 +96,10 @@ def find_all(root_node: ast.AST) -> Optional[List[str]]: def get_names_from_file( module: pathlib.Path, - modname: str, - package: str, - package_source: Source, - only_all: bool = False, underlined: bool = False, -) -> Generator[Name, None, None]: +) -> Generator[PartialName, None, None]: """ Get all the names from a given file using ast. - - Parameters - __________ - only_all: bool - only use __all__ to determine the module's contents """ with open(module, mode="rb") as file: try: @@ -118,18 +107,14 @@ def get_names_from_file( except SyntaxError as error: print(error) return - all = find_all(root_node) for node in ast.iter_child_nodes(root_node): if isinstance(node, ast.Assign): for target in node.targets: try: assert isinstance(target, ast.Name) if underlined or not target.id.startswith("_"): - yield Name( + yield PartialName( target.id, - modname, - package, - package_source, get_type_ast(node), ) except (AttributeError, AssertionError): @@ -137,60 +122,12 @@ def get_names_from_file( pass elif isinstance(node, (ast.FunctionDef, ast.ClassDef)): if underlined or not node.name.startswith("_"): - yield Name( + yield PartialName( node.name, - modname, - package, - package_source, get_type_ast(node), ) -def find_all_names_in_package( - package_path: pathlib.Path, - recursive=True, - package_source: Source = None, - underlined: bool = False, -) -> List[Name]: - """ - Find all names in a package. - - Parameters - ---------- - package_path : pathlib.Path - path to the package - recursive : bool - scan submodules in addition to the root directory - underlined : bool - include underlined directories - """ - package_tuple = get_package_name_from_path(package_path) - if package_tuple is None: - return [] - package_name, package_type = package_tuple - if package_source is None: - package_source = get_package_source(package_path) - modules: List[Tuple[pathlib.Path, str]] = [] - if package_type is PackageType.SINGLE_FILE: - modules.append((package_path, package_name)) - elif package_type is PackageType.COMPILED: - return [] - elif recursive: - for sub in submodules(package_path): - modname = get_modname_from_path(sub, package_path) - if underlined or modname.__contains__("_"): - continue # Exclude private items - modules.append((sub, modname)) - else: - modules.append((package_path, package_name)) - result: List[Name] = [] - for module in modules: - result.extend( - get_names(module[0], module[1], package_name, package_source, underlined) - ) - return result - - def get_type_object(object) -> NameType: if inspect.isclass(object): return NameType.Class diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index fbf1060fb..bfa3aa790 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -2,16 +2,16 @@ import pathlib import sys from collections import OrderedDict -from typing import List, Optional, Set, Tuple +from typing import Generator, List, Optional, Set, Tuple from rope.base.project import Project -from .defs import PackageType, Source +from .defs import NameFile, Package, PackageType, Source -def get_package_name_from_path( - package_path: pathlib.Path, -) -> Optional[Tuple[str, PackageType]]: +def get_package_tuple( + package_path: pathlib.Path, project: Optional[Project] = None +) -> Optional[Package]: """ Get package name and type from a path. @@ -19,19 +19,35 @@ def get_package_name_from_path( Returns None if not a viable package. """ package_name = package_path.name + package_source = get_package_source(package_path, project) if package_name.startswith(".") or package_name == "__pycache__": return None if package_path.is_file(): if package_name.endswith(".so"): name = package_name.split(".")[0] - return (name, PackageType.COMPILED) + return Package(name, package_source, package_path, PackageType.COMPILED) if package_name.endswith(".py"): stripped_name = package_path.stem - return (stripped_name, PackageType.SINGLE_FILE) + return Package( + stripped_name, package_source, package_path, PackageType.SINGLE_FILE + ) return None if package_name.endswith((".egg-info", ".dist-info")): return None - return (package_name, PackageType.STANDARD) + return Package(package_name, package_source, package_path, PackageType.STANDARD) + + +def get_package_source( + package: pathlib.Path, project: Optional[Project] = None +) -> Source: + """Detect the source of a given package. Rudimentary implementation.""" + if project is not None and package.as_posix().__contains__(project.address): + return Source.PROJECT + if package.as_posix().__contains__("site-packages"): + return Source.SITE_PACKAGE + if package.as_posix().startswith(sys.prefix): + return Source.STANDARD + return Source.UNKNOWN def get_modname_from_path( @@ -56,19 +72,6 @@ def get_modname_from_path( return modname -def get_package_source( - package: pathlib.Path, project: Optional[Project] = None -) -> Source: - """Detect the source of a given package. Rudimentary implementation.""" - if project is not None and package.as_posix().__contains__(project.address): - return Source.PROJECT - if package.as_posix().__contains__("site-packages"): - return Source.SITE_PACKAGE - if package.as_posix().startswith(sys.prefix): - return Source.STANDARD - return Source.UNKNOWN - - def sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: """Sort and deduplicate a list of name, source entries.""" if len(results) == 0: @@ -99,3 +102,28 @@ def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: for child in mod.iterdir(): result |= submodules(child) return result + + +def get_files( + package: Package, underlined: bool = False +) -> Generator[NameFile, None, None]: + """Find all files to parse in a given path using __init__.py.""" + if package.type == PackageType.SINGLE_FILE: + assert package.path.suffix == ".py" + yield NameFile(package.path, package.path.stem, underlined) + for folder in submodules(package.path): + for file in folder.iterdir(): + if file.suffix == ".py": + if file.name == "__init__.py": + yield NameFile( + file, + get_modname_from_path(folder, package.path), + underlined, + process_imports = True, + ) + else: + yield NameFile( + file, + get_modname_from_path(file, package.path), + underlined, + ) diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index b6e9fcb0d..c799cc1c4 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -2,34 +2,17 @@ from typing import Dict from rope.contrib.autoimport import parse -from rope.contrib.autoimport.defs import Name, NameType, PackageType, Source +from rope.contrib.autoimport.defs import Name, NameType, PartialName, Source def test_typing_names(typing_path): - names = list( - parse.get_names_from_file( - typing_path, typing_path.stem, typing_path, Source.STANDARD - ) - ) + names = list(parse.get_names_from_file(typing_path)) print(names) - assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( - names - ) + assert PartialName("Dict", NameType.Class) in list(names) -def test_get_typing_names(typing_path): - names = parse.get_names(typing_path, typing_path.stem, typing_path, Source.STANDARD) - assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( - names - ) -def test_find_all_typing_names(typing_path): - names = parse.find_all_names_in_package(typing_path) - assert Name("Dict", "typing", "typing", Source.STANDARD, NameType.Class) in list( - names - ) - def test_find_sys(): names = list(parse.get_names_from_compiled("sys", Source.BUILTIN)) diff --git a/ropetest/contrib/autoimport/utilstest.py b/ropetest/contrib/autoimport/utilstest.py index 5b2d3e4df..423c0e754 100644 --- a/ropetest/contrib/autoimport/utilstest.py +++ b/ropetest/contrib/autoimport/utilstest.py @@ -2,7 +2,7 @@ import pathlib from rope.contrib.autoimport import utils -from rope.contrib.autoimport.defs import PackageType, Source +from rope.contrib.autoimport.defs import Package, PackageType, Source def test_get_package_source(mod1_path, project): @@ -40,19 +40,20 @@ def test_get_modname_folder(build_path, build_env_path): assert utils.get_modname_from_path(build_env_path, build_path) == "build.env" -def test_get_package_name_sample(project_path): - package_name, package_type = utils.get_package_name_from_path(project_path) - assert package_name == "sample_project" - assert package_type == PackageType.STANDARD +def test_get_package_tuple_sample(project_path): + assert Package( + "sample_project", Source.UNKNOWN, project_path, PackageType.STANDARD + ) == utils.get_package_tuple(project_path) -def test_get_package_name_typing(typing_path): - package_name, package_type = utils.get_package_name_from_path(typing_path) - assert package_name == "typing" - assert package_type == PackageType.SINGLE_FILE +def test_get_package_tuple_typing(typing_path): + assert Package( + "typing", Source.STANDARD, typing_path, PackageType.SINGLE_FILE + ) == utils.get_package_tuple(typing_path) -def test_get_package_name_compiled(zlib_path): - package_name, package_type = utils.get_package_name_from_path(zlib_path) - assert package_name == "zlib" - assert package_type == PackageType.COMPILED + +def test_get_package_tuple_compiled(zlib_path): + assert Package( + "zlib", Source.STANDARD, zlib_path, PackageType.COMPILED + ) == utils.get_package_tuple(zlib_path) From 9f094101d68e90286c18d6bda0eaa9e1a2d8b620 Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 00:56:29 -0500 Subject: [PATCH 47/62] WIP on autoimport using namedTuples --- rope/contrib/autoimport/autoimport.py | 94 ++++++++++++++++----------- rope/contrib/autoimport/parse.py | 7 +- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index df0637562..95da4b6f0 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -7,17 +7,17 @@ from collections import OrderedDict from concurrent.futures import ProcessPoolExecutor from itertools import chain, repeat -from typing import List, Optional, Tuple, Iterable +from typing import Iterable, List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource from rope.contrib.autoimport.defs import Name, Package, PackageType, Source -from rope.contrib.autoimport.parse import ( - get_names_from_compiled, +from rope.contrib.autoimport.parse import (combine, get_names_from_compiled, get_names_from_file) -from rope.contrib.autoimport.utils import (get_modname_from_path, +from rope.contrib.autoimport.utils import (get_files, get_modname_from_path, get_package_source, + get_package_tuple, sort_and_deduplicate, sort_and_deduplicate_tuple) from rope.refactor import importutils @@ -229,15 +229,12 @@ def generate_modules_cache( Do not use this for generating your own project's internal names, use generate_resource_cache for that instead. """ - packages: List[pathlib.Path] = [] + packages: List[Package] = [] compiled_packages: List[Tuple[str, Source]] = [] - to_add: List[Package] = [] if self.underlined: underlined = True if modules is None: - packages, compiled_packages, to_add = self._get_avalible_packages( - underlined - ) + packages, compiled_packages = self._get_avalible_packages(underlined) else: existing = self._get_existing() for modname in modules: @@ -265,16 +262,25 @@ def generate_modules_cache( self._add_packages(to_add) if single_thread: for package in packages: - self._add_names( - find_all_names_in_package(package, underlined=underlined) - ) + for module in get_files(package, underlined): + for name in get_names_from_file(package.path, underlined): + self._add_name(combine(package, module, name)) + else: underlined_list = repeat(underlined, len(packages)) - with ProcessPoolExecutor() as exectuor: - for name_list in exectuor.map( - find_all_names_in_package, packages, underlined_list + with ProcessPoolExecutor() as executor: + for package, modules in zip( + packages, executor.map(get_files, packages, underlined_list) ): - self._add_names(name_list) + modules = list(modules) + underlined_list = repeat(underlined, len(modules)) + for names, module in zip( + executor.map(get_names_from_file, modules, underlined_list), + modules, + ): + for name in names: + self._add_name(combine(package, module, name)) + for compiled_package, source in compiled_packages: try: self._add_names( @@ -361,19 +367,24 @@ def update_resource( resource_modname: str = get_modname_from_path( resource_path, package_path, add_package_name=False ) - package_tuple = get_package_name_from_path(package_path) + package_tuple = get_package_tuple(package_path) if package_tuple is None: return package_name = package_tuple[0] self._del_if_exist(module_name=resource_modname, commit=False) - names = get_names_from_file( + for partial in get_names_from_file( resource_path, - resource_modname, - package_name, - Source.PROJECT, underlined=underlined, - ) - self._add_names(names) + ): + self._add_name( + Name( + partial.name, + resource_modname, + package_name, + Source.PROJECT, + partial.name_type, + ) + ) if commit: self.connection.commit() @@ -401,11 +412,8 @@ def _get_python_folders(self) -> List[pathlib.Path]: def _get_avalible_packages( self, underlined: bool = False - ) -> Tuple[List[pathlib.Path], List[Tuple[str, Source]], List[Package]]: - packages: List[pathlib.Path] = [] - package_names: List[ - Package - ] = [] # List of packages to add to the package table + ) -> Tuple[List[Package], List[Tuple[str, Source]]]: + packages: List[Package] = [] # Get builtins first compiled_packages: List[Tuple[str, Source]] = [ (module, Source.BUILTIN) for module in sys.builtin_module_names @@ -414,10 +422,10 @@ def _get_avalible_packages( underlined = underlined if underlined else self.underlined for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_name_from_path(package) + package_tuple = get_package_tuple(package) if package_tuple is None: continue - package_name, package_type = package_tuple + package_name, source, _, package_type = package_tuple if package_name in existing: continue if package_name.startswith("_") and not underlined: @@ -427,12 +435,12 @@ def _get_avalible_packages( (package_name, get_package_source(package, self.project)) ) else: - packages.append(package) - package_names.append((package_name,)) - return packages, compiled_packages, package_names + packages.append(package_tuple) + return packages, compiled_packages def _add_packages(self, packages: List[Package]): - self.connection.executemany("INSERT into packages values(?)", packages) + for package in packages: + self.connection.execute("INSERT into packages values(?)", (package.name,)) def _get_existing(self) -> List[str]: existing: List[str] = list( @@ -444,7 +452,7 @@ def _get_existing(self) -> List[str]: @property def _project_name(self): package_path: pathlib.Path = pathlib.Path(self.project.address) - package_tuple = get_package_name_from_path(package_path) + package_tuple = get_package_tuple(package_path) if package_tuple is None: return None return package_tuple[0] @@ -472,6 +480,18 @@ def _add_names(self, names: Iterable[Name]): names, ) + def _add_name(self, name: Name): + self.connection.execute( + "insert into names values (?,?,?,?,?)", + ( + name.name, + name.modname, + name.package, + name.source.value, + name.name_type.value, + ), + ) + def _check_import(self, module: pathlib.Path) -> bool: """ Check the ability to import an external package, removes it if not avalible. @@ -497,10 +517,10 @@ def _find_package_path( return (None, target_name, PackageType.BUILTIN) for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_name_from_path(package) + package_tuple = get_package_tuple(package) if package_tuple is None: continue - name, package_type = package_tuple + name, _, _, package_type = package_tuple if name == target_name: return (package, name, package_type) return None diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 6f2564f60..7deff3800 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -10,7 +10,7 @@ from importlib import import_module from typing import Generator, List, Optional -from .defs import Name, NameType, PartialName, Source +from .defs import Name, NameFile, NameType, Package, PartialName, Source # def get_names( # modpath: pathlib.Path, @@ -173,3 +173,8 @@ def get_names_from_compiled( yield Name( str(name), package, package, source, get_type_object(value) ) + + +def combine(package: Package, module: NameFile, name: PartialName) -> Name: + """Combine the information from a package, module, and partial name to form a full name.""" + return Name(name.name, module.modname, package.name, package.source, name.name_type) From 780b52b8c84396aa2fcafddfda299b54254acd7b Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 11:46:30 -0500 Subject: [PATCH 48/62] mostly working threaded implementation --- rope/contrib/autoimport/autoimport.py | 185 ++++++++++---------------- rope/contrib/autoimport/defs.py | 22 ++- rope/contrib/autoimport/parse.py | 85 ++++-------- rope/contrib/autoimport/utils.py | 56 ++++---- 4 files changed, 141 insertions(+), 207 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 95da4b6f0..1b5d38d7c 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -1,28 +1,47 @@ """AutoImport module for rope.""" -import logging import pathlib import re import sqlite3 import sys from collections import OrderedDict -from concurrent.futures import ProcessPoolExecutor -from itertools import chain, repeat +from concurrent.futures import Future, ProcessPoolExecutor, as_completed +from itertools import chain from typing import Iterable, List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource -from rope.contrib.autoimport.defs import Name, Package, PackageType, Source -from rope.contrib.autoimport.parse import (combine, get_names_from_compiled, - get_names_from_file) +from rope.contrib.autoimport.defs import (ModuleFile, Name, Package, + PackageType, Source) +from rope.contrib.autoimport.parse import get_names from rope.contrib.autoimport.utils import (get_files, get_modname_from_path, - get_package_source, get_package_tuple, sort_and_deduplicate, sort_and_deduplicate_tuple) from rope.refactor import importutils -logger = logging.getLogger(__name__) + +def get_future_names(packages: List[Package], underlined: bool): + with ProcessPoolExecutor() as executor: + for package in packages: + for module in get_files(package, underlined): + yield executor.submit(get_names, module, package) + + +def filter_packages( + packages: Iterable[Package], underlined: bool, existing: List[str] +) -> Iterable[Package]: + if underlined: + + def filter_package(package: Package) -> bool: + return package.name not in existing + + else: + + def filter_package(package: Package) -> bool: + return package.name not in existing and not package.name.startswith("_") + + return filter(filter_package, packages) class AutoImport: @@ -216,7 +235,7 @@ def generate_resource_cache( def generate_modules_cache( self, - modules: List[str] = None, + modules_to_find: List[str] = None, task_handle=taskhandle.NullTaskHandle(), single_thread: bool = False, underlined: bool = False, @@ -230,66 +249,30 @@ def generate_modules_cache( use generate_resource_cache for that instead. """ packages: List[Package] = [] - compiled_packages: List[Tuple[str, Source]] = [] if self.underlined: underlined = True - if modules is None: - packages, compiled_packages = self._get_avalible_packages(underlined) + existing = self._get_existing() + if modules_to_find is None: + packages = self._get_avalible_packages() else: - existing = self._get_existing() - for modname in modules: - mod_tuple = self._find_package_path(modname) - if mod_tuple is None: - continue - package_path, package_name, package_type = mod_tuple - if package_name.startswith("_") and not underlined: + for modname in modules_to_find: + package = self._find_package_path(modname) + if package is None: continue - if package_name in existing: - continue - if package_type in (PackageType.COMPILED, PackageType.BUILTIN): - if package_type is PackageType.COMPILED: - assert ( - package_path is not None - ) # It should have been found, and isn't a builtin - source = get_package_source(package_path, self.project) - else: - source = Source.BUILTIN - compiled_packages.append((package_name, source)) - else: - assert package_path # Should only return none for a builtin - packages.append(package_path) - to_add.append((package_name,)) - self._add_packages(to_add) + packages.append(package) + packages = list(filter_packages(packages,underlined, existing)) + self._add_packages(packages) if single_thread: for package in packages: - for module in get_files(package, underlined): - for name in get_names_from_file(package.path, underlined): - self._add_name(combine(package, module, name)) - + if package.type in (PackageType.BUILTIN, PackageType.COMPILED): + for module in get_files(package, underlined): + for name in get_names(module, package): + self._add_name(name) else: - underlined_list = repeat(underlined, len(packages)) - with ProcessPoolExecutor() as executor: - for package, modules in zip( - packages, executor.map(get_files, packages, underlined_list) - ): - modules = list(modules) - underlined_list = repeat(underlined, len(modules)) - for names, module in zip( - executor.map(get_names_from_file, modules, underlined_list), - modules, - ): - for name in names: - self._add_name(combine(package, module, name)) - - for compiled_package, source in compiled_packages: - try: - self._add_names( - get_names_from_compiled(compiled_package, source, underlined) - ) - except Exception as e: - logger.error( - f"{compiled_package} could not be imported for autoimport analysis" - ) + + for future_name in as_completed(get_future_names(packages, underlined)): + self._add_names(future_name.result()) + self.connection.commit() def update_module(self, module: str): @@ -358,33 +341,21 @@ def update_resource( self, resource: Resource, underlined: bool = False, commit: bool = True ): """Update the cache for global names in `resource`.""" - resource_path: pathlib.Path = pathlib.Path(resource.real_path) - package_path: pathlib.Path = self._project_path underlined = underlined if underlined else self.underlined + package = get_package_tuple(self._project_path, self.project) + if package is None or package.path is None: + return + resource_path: pathlib.Path = pathlib.Path(resource.real_path) # The project doesn't need its name added to the path, # since the standard python file layout accounts for that # so we set add_package_name to False resource_modname: str = get_modname_from_path( - resource_path, package_path, add_package_name=False + resource_path, package.path, add_package_name=False ) - package_tuple = get_package_tuple(package_path) - if package_tuple is None: - return - package_name = package_tuple[0] + module = ModuleFile(resource_path, resource_modname, underlined) self._del_if_exist(module_name=resource_modname, commit=False) - for partial in get_names_from_file( - resource_path, - underlined=underlined, - ): - self._add_name( - Name( - partial.name, - resource_modname, - package_name, - Source.PROJECT, - partial.name_type, - ) - ) + for name in get_names(module, package): + self._add_name(name) if commit: self.connection.commit() @@ -410,33 +381,19 @@ def _get_python_folders(self) -> List[pathlib.Path]: ] return list(OrderedDict.fromkeys(folder_paths)) - def _get_avalible_packages( - self, underlined: bool = False - ) -> Tuple[List[Package], List[Tuple[str, Source]]]: - packages: List[Package] = [] - # Get builtins first - compiled_packages: List[Tuple[str, Source]] = [ - (module, Source.BUILTIN) for module in sys.builtin_module_names + def _get_avalible_packages(self) -> List[Package]: + packages: List[Package] = [ + Package(module, Source.BUILTIN, None, PackageType.BUILTIN) + for module in sys.builtin_module_names ] - existing = self._get_existing() - underlined = underlined if underlined else self.underlined for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_tuple(package) + package_tuple = get_package_tuple(package, self.project) if package_tuple is None: continue package_name, source, _, package_type = package_tuple - if package_name in existing: - continue - if package_name.startswith("_") and not underlined: - continue - if package_type == PackageType.COMPILED: - compiled_packages.append( - (package_name, get_package_source(package, self.project)) - ) - else: - packages.append(package_tuple) - return packages, compiled_packages + packages.append(package_tuple) + return packages def _add_packages(self, packages: List[Package]): for package in packages: @@ -474,11 +431,12 @@ def _removed(self, resource): modname = self._modname(resource) self._del_if_exist(modname) + def _add_future_names(self, names: Future[List[Name]]): + self._add_names(names.result()) + def _add_names(self, names: Iterable[Name]): - self.connection.executemany( - "insert into names(name,module,package,source) values (?,?,?,?,?)", - names, - ) + for name in names: + self._add_name(name) def _add_name(self, name: Name): self.connection.execute( @@ -510,17 +468,16 @@ def _check_all(self): """Check all modules and removes bad ones.""" pass - def _find_package_path( - self, target_name: str - ) -> Optional[Tuple[Optional[pathlib.Path], str, PackageType]]: + def _find_package_path(self, target_name: str) -> Optional[Package]: if target_name in sys.builtin_module_names: - return (None, target_name, PackageType.BUILTIN) + return Package(target_name, Source.BUILTIN, None, PackageType.BUILTIN) for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_tuple(package) + package_tuple = get_package_tuple(package, self.project) if package_tuple is None: continue - name, _, _, package_type = package_tuple + name, source, package_path, package_type = package_tuple if name == target_name: - return (package, name, package_type) + return package_tuple + return None diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 440ae4d44..3d2df0cdd 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -1,7 +1,7 @@ """Definitions of types for the Autoimport program.""" import pathlib from enum import Enum -from typing import NamedTuple +from typing import NamedTuple, Optional class Source(Enum): @@ -17,7 +17,16 @@ class Source(Enum): # modified_time -class NameFile(NamedTuple): +class ModuleInfo(NamedTuple): + """Descriptor of information to get names from a file.""" + + filepath: Optional[pathlib.Path] + modname: str + underlined: bool + process_imports: bool = False + + +class ModuleFile(ModuleInfo): """Descriptor of information to get names from a file using ast.""" filepath: pathlib.Path @@ -26,6 +35,13 @@ class NameFile(NamedTuple): process_imports: bool = False +class ModuleCompiled(ModuleInfo): + filepath = None + modname: str + underlined: bool + process_imports: bool = False + + class PackageType(Enum): """Describes the type of package, to determine how to get the names from it.""" @@ -68,7 +84,7 @@ class NameType(Enum): class Package(NamedTuple): name: str source: Source - path: pathlib.Path + path: Optional[pathlib.Path] type: PackageType diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 7deff3800..128ed0e29 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -6,51 +6,15 @@ import ast import inspect +import logging import pathlib from importlib import import_module -from typing import Generator, List, Optional - -from .defs import Name, NameFile, NameType, Package, PartialName, Source - -# def get_names( -# modpath: pathlib.Path, -# modname: str, -# package_name: str, -# package_source: Source, -# underlined: bool = False, -# ) -> Generator[Name, None, None]: -# """Get all names in the `modname` module, located at modpath. -# -# `modname` is the name of a module. -# """ -# if modpath.is_dir(): -# names: List[Name] -# if (modpath / "__init__.py").exists(): -# names = get_names_from_file( -# modpath / "__init__.py", -# modname, -# package_name, -# package_source, -# only_all=True, -# ) -# if len(names) > 0: -# return names -# names = [] -# for file in modpath.glob("*.py"): -# names.extend( -# get_names_from_file( -# file, -# modname + f".{file.stem}", -# package_name, -# package_source, -# underlined=underlined, -# ) -# ) -# return names -# if modpath.suffix == ".py": -# return get_names_from_file( -# modpath, modname, package_name, package_source, underlined=underlined -# ) +from typing import Generator, List + +from .defs import (ModuleCompiled, ModuleFile, ModuleInfo, Name, NameType, + Package, PartialName, Source) + +logger = logging.getLogger(__name__) def parse_all( @@ -77,23 +41,6 @@ def get_type_ast(node: ast.AST) -> NameType: return NameType.Text # default value -def find_all(root_node: ast.AST) -> Optional[List[str]]: - """Find the contents of __all__.""" - for node in ast.iter_child_nodes(root_node): - if isinstance(node, ast.Assign): - for target in node.targets: - try: - assert isinstance(target, ast.Name) - if target.id == "__all__": - assert isinstance(node.value, ast.List) - assert node.value.elts is not None - return node.value.elts - except (AttributeError, AssertionError): - # TODO handle tuple assignment - pass - return None - - def get_names_from_file( module: pathlib.Path, underlined: bool = False, @@ -136,6 +83,20 @@ def get_type_object(object) -> NameType: return NameType.Constant +def get_names(module: ModuleInfo, package: Package) -> List[Name]: + """Get all names from a module and package.""" + if isinstance(module, ModuleCompiled): + return list( + get_names_from_compiled(package.name, package.source, module.underlined) + ) + elif isinstance(module, ModuleFile): + return [ + combine(package, module, partial_name) + for partial_name in get_names_from_file(module.filepath, module.underlined) + ] + return [] + + def get_names_from_compiled( package: str, source: Source, @@ -160,7 +121,7 @@ def get_names_from_compiled( try: module = import_module(str(package)) except ImportError: - # print(f"couldn't import {package}") + logger.error(f"{package} could not be imported for autoimport analysis") return else: for name, value in inspect.getmembers(module): @@ -175,6 +136,6 @@ def get_names_from_compiled( ) -def combine(package: Package, module: NameFile, name: PartialName) -> Name: +def combine(package: Package, module: ModuleFile, name: PartialName) -> Name: """Combine the information from a package, module, and partial name to form a full name.""" return Name(name.name, module.modname, package.name, package.source, name.name_type) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index bfa3aa790..35fc8fa89 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -6,7 +6,8 @@ from rope.base.project import Project -from .defs import NameFile, Package, PackageType, Source +from .defs import (ModuleCompiled, ModuleFile, ModuleInfo, Package, + PackageType, Source) def get_package_tuple( @@ -94,36 +95,35 @@ def sort_and_deduplicate_tuple( return list(OrderedDict.fromkeys(results_sorted)) -def submodules(mod: pathlib.Path) -> Set[pathlib.Path]: - """Find submodules in a given path using __init__.py.""" - result = set() - if mod.is_dir() and (mod / "__init__.py").exists(): - result.add(mod) - for child in mod.iterdir(): - result |= submodules(child) - return result + + +def get_files_list(*args) -> List[ModuleInfo]: + return list(get_files(*args)) def get_files( package: Package, underlined: bool = False -) -> Generator[NameFile, None, None]: +) -> Generator[ModuleInfo, None, None]: """Find all files to parse in a given path using __init__.py.""" - if package.type == PackageType.SINGLE_FILE: + if package.type in (PackageType.COMPILED, PackageType.BUILTIN): + yield ModuleCompiled(None, package.name, underlined, process_imports=True) + elif package.type == PackageType.SINGLE_FILE: + assert package.path assert package.path.suffix == ".py" - yield NameFile(package.path, package.path.stem, underlined) - for folder in submodules(package.path): - for file in folder.iterdir(): - if file.suffix == ".py": - if file.name == "__init__.py": - yield NameFile( - file, - get_modname_from_path(folder, package.path), - underlined, - process_imports = True, - ) - else: - yield NameFile( - file, - get_modname_from_path(file, package.path), - underlined, - ) + yield ModuleFile(package.path, package.path.stem, underlined) + else: + assert package.path + for file in package.path.glob("*.py"): + if file.name == "__init__.py": + yield ModuleFile( + file, + get_modname_from_path(file.parent, package.path), + underlined, + process_imports=True, + ) + else: + yield ModuleFile( + file, + get_modname_from_path(file, package.path), + underlined, + ) From 7035b6e21a6138779561d23654971479722cbf84 Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 13:30:31 -0500 Subject: [PATCH 49/62] reformat and update tests --- rope/contrib/autoimport/autoimport.py | 37 ++++++++++++++---------- rope/contrib/autoimport/defs.py | 12 +++++--- rope/contrib/autoimport/parse.py | 28 +++++------------- rope/contrib/autoimport/utils.py | 8 +---- ropetest/contrib/autoimport/parsetest.py | 9 +----- ropetest/contrib/autoimporttest.py | 23 +++++---------- 6 files changed, 46 insertions(+), 71 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 1b5d38d7c..36402c9f8 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -6,7 +6,7 @@ from collections import OrderedDict from concurrent.futures import Future, ProcessPoolExecutor, as_completed from itertools import chain -from typing import Iterable, List, Optional, Tuple +from typing import Generator, Iterable, List, Optional, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project @@ -21,7 +21,10 @@ from rope.refactor import importutils -def get_future_names(packages: List[Package], underlined: bool): +def get_future_names( + packages: List[Package], underlined: bool +) -> Generator[Future[Iterable[Name]], None, None]: + """Get all names as futures.""" with ProcessPoolExecutor() as executor: for package in packages: for module in get_files(package, underlined): @@ -31,6 +34,7 @@ def get_future_names(packages: List[Package], underlined: bool): def filter_packages( packages: Iterable[Package], underlined: bool, existing: List[str] ) -> Iterable[Package]: + """Filter list of packages to parse.""" if underlined: def filter_package(package: Package) -> bool: @@ -161,13 +165,13 @@ def lsp_search( name = name + "%" # Makes the query a starts_with query results_name: List[Tuple[str, str, int, int]] = [] results_module: List[Tuple[str, str, int, int]] = [] - for import_name, module, source, type in self.connection.execute( + for import_name, module, source, name_type in self.connection.execute( "SELECT name, module, source, type FROM names WHERE name LIKE (?)", (name,) ): results_name.append( - (f"from {module} import {import_name}", import_name, source, type) + (f"from {module} import {import_name}", import_name, source, name_type) ) - for module, source, type in self.connection.execute( + for module, source, name_type in self.connection.execute( "Select module, source, type FROM names where module LIKE (?)", ("%." + name,), ): @@ -178,12 +182,17 @@ def lsp_search( remaining += "." remaining += part results_module.append( - (f"from {remaining} import {import_name}", import_name, source, type) + ( + f"from {remaining} import {import_name}", + import_name, + source, + name_type, + ) ) - for module, source, type in self.connection.execute( + for module, source, name_type in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): - results_module.append((f"import {module}", module, source, type)) + results_module.append((f"import {module}", module, source, name_type)) return results_name, results_module def get_modules(self, name) -> List[str]: @@ -236,7 +245,6 @@ def generate_resource_cache( def generate_modules_cache( self, modules_to_find: List[str] = None, - task_handle=taskhandle.NullTaskHandle(), single_thread: bool = False, underlined: bool = False, ): @@ -260,16 +268,14 @@ def generate_modules_cache( if package is None: continue packages.append(package) - packages = list(filter_packages(packages,underlined, existing)) + packages = list(filter_packages(packages, underlined, existing)) self._add_packages(packages) if single_thread: for package in packages: - if package.type in (PackageType.BUILTIN, PackageType.COMPILED): - for module in get_files(package, underlined): - for name in get_names(module, package): - self._add_name(name) + for module in get_files(package, underlined): + for name in get_names(module, package): + self._add_name(name) else: - for future_name in as_completed(get_future_names(packages, underlined)): self._add_names(future_name.result()) @@ -391,7 +397,6 @@ def _get_avalible_packages(self) -> List[Package]: package_tuple = get_package_tuple(package, self.project) if package_tuple is None: continue - package_name, source, _, package_type = package_tuple packages.append(package_tuple) return packages diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 3d2df0cdd..51e31df2d 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -18,7 +18,7 @@ class Source(Enum): class ModuleInfo(NamedTuple): - """Descriptor of information to get names from a file.""" + """Descriptor of information to get names from a module.""" filepath: Optional[pathlib.Path] modname: str @@ -36,6 +36,8 @@ class ModuleFile(ModuleInfo): class ModuleCompiled(ModuleInfo): + """Descriptor of information to get names using imports.""" + filepath = None modname: str underlined: bool @@ -52,7 +54,7 @@ class PackageType(Enum): class NameType(Enum): - """Describes the type of Name for lsp completions. Taken from python lsp server""" + """Describes the type of Name for lsp completions. Taken from python lsp server.""" Text = 1 Method = 2 @@ -82,6 +84,8 @@ class NameType(Enum): class Package(NamedTuple): + """Attributes of a package.""" + name: str source: Source path: Optional[pathlib.Path] @@ -89,7 +93,7 @@ class Package(NamedTuple): class Name(NamedTuple): - """A Name to be added to the database""" + """A Name to be added to the database.""" name: str modname: str @@ -99,7 +103,7 @@ class Name(NamedTuple): class PartialName(NamedTuple): - """Partial information of a Name""" + """Partial information of a Name.""" name: str name_type: NameType diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 128ed0e29..a4f5fbcf7 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -17,19 +17,6 @@ logger = logging.getLogger(__name__) -def parse_all( - node: ast.Assign, modname: str, package: str, package_source: Source -) -> Generator[Name, None, None]: - """Parse the node which contains the value __all__ and return its contents.""" - # I assume that the __all__ value isn't assigned via tuple - assert isinstance(node.value, ast.List) - for item in node.value.elts: - assert isinstance(item, ast.Constant) - name_type: NameType = NameType.Keyword - # TODO somehow determine the actual value of this since every member of all is a string - yield Name(str(item.value), modname, package, package_source, name_type) - - def get_type_ast(node: ast.AST) -> NameType: """Get the lsp type of a node.""" if isinstance(node, ast.ClassDef): @@ -45,9 +32,7 @@ def get_names_from_file( module: pathlib.Path, underlined: bool = False, ) -> Generator[PartialName, None, None]: - """ - Get all the names from a given file using ast. - """ + """Get all the names from a given file using ast.""" with open(module, mode="rb") as file: try: root_node = ast.parse(file.read()) @@ -75,10 +60,11 @@ def get_names_from_file( ) -def get_type_object(object) -> NameType: - if inspect.isclass(object): +def get_type_object(imported_object) -> NameType: + """Determine the type of an object.""" + if inspect.isclass(imported_object): return NameType.Class - if inspect.isfunction(object) or inspect.isbuiltin(object): + if inspect.isfunction(imported_object) or inspect.isbuiltin(imported_object): return NameType.Function return NameType.Constant @@ -89,7 +75,7 @@ def get_names(module: ModuleInfo, package: Package) -> List[Name]: return list( get_names_from_compiled(package.name, package.source, module.underlined) ) - elif isinstance(module, ModuleFile): + if isinstance(module, ModuleFile): return [ combine(package, module, partial_name) for partial_name in get_names_from_file(module.filepath, module.underlined) @@ -137,5 +123,5 @@ def get_names_from_compiled( def combine(package: Package, module: ModuleFile, name: PartialName) -> Name: - """Combine the information from a package, module, and partial name to form a full name.""" + """Combine information to form a full name.""" return Name(name.name, module.modname, package.name, package.source, name.name_type) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 35fc8fa89..2407b484a 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -2,7 +2,7 @@ import pathlib import sys from collections import OrderedDict -from typing import Generator, List, Optional, Set, Tuple +from typing import Generator, List, Optional, Tuple from rope.base.project import Project @@ -95,12 +95,6 @@ def sort_and_deduplicate_tuple( return list(OrderedDict.fromkeys(results_sorted)) - - -def get_files_list(*args) -> List[ModuleInfo]: - return list(get_files(*args)) - - def get_files( package: Package, underlined: bool = False ) -> Generator[ModuleInfo, None, None]: diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index c799cc1c4..e37db9cee 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -1,17 +1,10 @@ -from itertools import chain -from typing import Dict - from rope.contrib.autoimport import parse from rope.contrib.autoimport.defs import Name, NameType, PartialName, Source def test_typing_names(typing_path): names = list(parse.get_names_from_file(typing_path)) - print(names) - assert PartialName("Dict", NameType.Class) in list(names) - - - + assert PartialName("Dict", NameType.Variable) in names def test_find_sys(): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 400ff00aa..49a7c8b50 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -116,13 +116,13 @@ def test_handling_builtin_modules(self): self.assertTrue("sys" in self.importer.get_modules("exit")) def test_search_submodule(self): - self.importer.update_module("os") - import_statement = ("from os import path", "path") + self.importer.update_module("build") + import_statement = ("from build import env", "env") self.assertTrue( - import_statement in self.importer.search("path", exact_match=True) + import_statement in self.importer.search("env", exact_match=True) ) - self.assertTrue(import_statement in self.importer.search("pa")) - self.assertTrue(import_statement in self.importer.search("path")) + self.assertTrue(import_statement in self.importer.search("en")) + self.assertTrue(import_statement in self.importer.search("env")) def test_search_module(self): self.importer.update_module("os") @@ -145,7 +145,9 @@ def test_search(self): self.assertTrue(import_statement in self.importer.search("D")) def test_generate_full_cache(self): - self.importer.generate_modules_cache() + """The single thread test takes much longer than the multithread test but is easier to debug""" + single_thread = False + self.importer.generate_modules_cache(single_thread=single_thread) self.assertTrue( ("from typing import Dict", "Dict") in self.importer.search("Dict") ) @@ -153,13 +155,6 @@ def test_generate_full_cache(self): for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) - # def test_generate_full_cache_st(self): - # """The single thread test takes much longer than the multithread test but is easier to debug""" - # self.importer.generate_modules_cache(single_thread=True) - # self.assertTrue("from typing import Dict" in self.importer.search("Dict")) - # self.assertTrue(len(self.importer._dump_all()) > 0) - # for table in self.importer._dump_all(): - # self.assertTrue(len(table) > 0) class AutoImportObservingTest(unittest.TestCase): @@ -188,5 +183,3 @@ def test_removing_files(self): self.mod1.write("myvar = None\n") self.mod1.remove() self.assertEqual([], self.importer.get_modules("myvar")) - - From 30595cd3450dc066e73a0697663947f078815db0 Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 13:31:05 -0500 Subject: [PATCH 50/62] blacken --- rope/contrib/autoimport/autoimport.py | 14 ++++++++------ rope/contrib/autoimport/parse.py | 12 ++++++++++-- rope/contrib/autoimport/utils.py | 3 +-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 36402c9f8..f64e37415 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -11,13 +11,15 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource -from rope.contrib.autoimport.defs import (ModuleFile, Name, Package, - PackageType, Source) +from rope.contrib.autoimport.defs import ModuleFile, Name, Package, PackageType, Source from rope.contrib.autoimport.parse import get_names -from rope.contrib.autoimport.utils import (get_files, get_modname_from_path, - get_package_tuple, - sort_and_deduplicate, - sort_and_deduplicate_tuple) +from rope.contrib.autoimport.utils import ( + get_files, + get_modname_from_path, + get_package_tuple, + sort_and_deduplicate, + sort_and_deduplicate_tuple, +) from rope.refactor import importutils diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index a4f5fbcf7..cee3b954b 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -11,8 +11,16 @@ from importlib import import_module from typing import Generator, List -from .defs import (ModuleCompiled, ModuleFile, ModuleInfo, Name, NameType, - Package, PartialName, Source) +from .defs import ( + ModuleCompiled, + ModuleFile, + ModuleInfo, + Name, + NameType, + Package, + PartialName, + Source, +) logger = logging.getLogger(__name__) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 2407b484a..b592556cf 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -6,8 +6,7 @@ from rope.base.project import Project -from .defs import (ModuleCompiled, ModuleFile, ModuleInfo, Package, - PackageType, Source) +from .defs import ModuleCompiled, ModuleFile, ModuleInfo, Package, PackageType, Source def get_package_tuple( From 292df317588eef2f53bed52729e36111fea81ba0 Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 13 Apr 2022 18:17:57 -0500 Subject: [PATCH 51/62] check underlined on submodules --- rope/contrib/autoimport/autoimport.py | 2 +- rope/contrib/autoimport/utils.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index f64e37415..9f6b73661 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -192,7 +192,7 @@ def lsp_search( ) ) for module, source, name_type in self.connection.execute( - "Select module, source from names where module LIKE (?)", (name,) + "Select module, source, type from names where module LIKE (?)", (name,) ): results_module.append((f"import {module}", module, source, name_type)) return results_name, results_module diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index b592556cf..b62cc881b 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -94,6 +94,15 @@ def sort_and_deduplicate_tuple( return list(OrderedDict.fromkeys(results_sorted)) +def should_parse(path: pathlib.Path, underlined: bool) -> bool: + if underlined: + return True + for part in path.parts: + if part.startswith("_"): + return False + return True + + def get_files( package: Package, underlined: bool = False ) -> Generator[ModuleInfo, None, None]: @@ -114,7 +123,7 @@ def get_files( underlined, process_imports=True, ) - else: + elif should_parse(file, underlined): yield ModuleFile( file, get_modname_from_path(file, package.path), From 7c4d3c283a9d74abdbe24bd362790cb60f69c647 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 14 Apr 2022 11:53:03 -0500 Subject: [PATCH 52/62] fix some typos --- .github/workflows/main.yml | 2 +- rope/contrib/autoimport/autoimport.py | 15 ++++++------ rope/contrib/autoimport/parse.py | 2 ++ rope/contrib/autoimport/utils.py | 3 ++- ropetest/contrib/autoimporttest.py | 35 +++++++++++---------------- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0e9ba5c7d..0f1503ead 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip -e .[dev] + python -m pip install -e .[dev] - name: Test with pytest run: | pytest -v diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 9f6b73661..fd3752bdc 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -93,7 +93,7 @@ def __init__(self, project, observe=True, underlined=False, memory=True): project.add_observer(observer) def _setup_db(self): - packages_table = "(pacakge TEXT)" + packages_table = "(package TEXT)" names_table = ( "(name TEXT, module TEXT, package TEXT, source INTEGER, type INTEGER)" ) @@ -220,7 +220,7 @@ def _dump_all(self) -> Tuple[List[Name], List[Package]]: package_results = self.connection.execute("select * from packages").fetchall() return name_results, package_results - def generate_resource_cache( + def generate_cache( self, resources: List[Resource] = None, underlined: bool = False, @@ -246,7 +246,8 @@ def generate_resource_cache( def generate_modules_cache( self, - modules_to_find: List[str] = None, + modules: List[str] = None, + task_handle=None, single_thread: bool = False, underlined: bool = False, ): @@ -262,10 +263,10 @@ def generate_modules_cache( if self.underlined: underlined = True existing = self._get_existing() - if modules_to_find is None: - packages = self._get_avalible_packages() + if modules is None: + packages = self._get_available_packages() else: - for modname in modules_to_find: + for modname in modules: package = self._find_package_path(modname) if package is None: continue @@ -389,7 +390,7 @@ def _get_python_folders(self) -> List[pathlib.Path]: ] return list(OrderedDict.fromkeys(folder_paths)) - def _get_avalible_packages(self) -> List[Package]: + def _get_available_packages(self) -> List[Package]: packages: List[Package] = [ Package(module, Source.BUILTIN, None, PackageType.BUILTIN) for module in sys.builtin_module_names diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index cee3b954b..24291f3bc 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -112,6 +112,8 @@ def get_names_from_compiled( banned = ["builtins", "python_crun"] if package in banned or (package.startswith("_") and not underlined): return # Builtins is redundant since you don't have to import it. + if source not in (Source.BUILTIN, Source.STANDARD, Source.PROJECT): + return try: module = import_module(str(package)) except ImportError: diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index b62cc881b..d73180988 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -108,7 +108,8 @@ def get_files( ) -> Generator[ModuleInfo, None, None]: """Find all files to parse in a given path using __init__.py.""" if package.type in (PackageType.COMPILED, PackageType.BUILTIN): - yield ModuleCompiled(None, package.name, underlined, process_imports=True) + if package.source in (Source.PROJECT, Source.STANDARD, Source.BUILTIN): + yield ModuleCompiled(None, package.name, underlined, process_imports=True) elif package.type == PackageType.SINGLE_FILE: assert package.path assert package.path.suffix == ".py" diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 49a7c8b50..76626ae1b 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -113,50 +113,43 @@ def test_name_locations_with_multiple_occurrences(self): def test_handling_builtin_modules(self): self.importer.update_module("sys") - self.assertTrue("sys" in self.importer.get_modules("exit")) + self.assertIn("sys", self.importer.get_modules("exit")) def test_search_submodule(self): self.importer.update_module("build") import_statement = ("from build import env", "env") - self.assertTrue( - import_statement in self.importer.search("env", exact_match=True) - ) - self.assertTrue(import_statement in self.importer.search("en")) - self.assertTrue(import_statement in self.importer.search("env")) + self.assertIn(import_statement, self.importer.search("env", exact_match=True)) + self.assertIn(import_statement, self.importer.search("en")) + self.assertIn(import_statement, self.importer.search("env")) def test_search_module(self): self.importer.update_module("os") import_statement = ("import os", "os") - self.assertTrue( - import_statement in self.importer.search("os", exact_match=True) + self.assertIn( + import_statement , self.importer.search("os", exact_match=True) ) - self.assertTrue(import_statement in self.importer.search("os")) - self.assertTrue(import_statement in self.importer.search("o")) + self.assertIn(import_statement, self.importer.search("os")) + self.assertIn(import_statement, self.importer.search("o")) def test_search(self): self.importer.update_module("typing") import_statement = ("from typing import Dict", "Dict") - self.assertTrue( - import_statement in self.importer.search("Dict", exact_match=True) - ) - self.assertTrue(import_statement in self.importer.search("Dict")) - self.assertTrue(import_statement in self.importer.search("Dic")) - self.assertTrue(import_statement in self.importer.search("Di")) - self.assertTrue(import_statement in self.importer.search("D")) + self.assertIn(import_statement, self.importer.search("Dict", exact_match=True)) + self.assertIn(import_statement, self.importer.search("Dict")) + self.assertIn(import_statement, self.importer.search("Dic")) + self.assertIn(import_statement, self.importer.search("Di")) + self.assertIn(import_statement, self.importer.search("D")) def test_generate_full_cache(self): """The single thread test takes much longer than the multithread test but is easier to debug""" single_thread = False self.importer.generate_modules_cache(single_thread=single_thread) - self.assertTrue( - ("from typing import Dict", "Dict") in self.importer.search("Dict") - ) + self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) self.assertTrue(len(self.importer._dump_all()) > 0) for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) - class AutoImportObservingTest(unittest.TestCase): def setUp(self): super(AutoImportObservingTest, self).setUp() From dea4f42702309c4e882a480c90fd0faffb8ded63 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 14 Apr 2022 16:18:42 -0500 Subject: [PATCH 53/62] fix: basic job set implementation, improve list comprehension of utils --- rope/contrib/autoimport/autoimport.py | 27 ++++++++++++++++----------- rope/contrib/autoimport/parse.py | 14 +++++++------- rope/contrib/autoimport/utils.py | 20 +++++++------------- ropetest/contrib/autoimporttest.py | 4 +--- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index fd3752bdc..ec4c0b6e8 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -11,25 +11,24 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource -from rope.contrib.autoimport.defs import ModuleFile, Name, Package, PackageType, Source +from rope.contrib.autoimport.defs import (ModuleFile, Name, Package, + PackageType, Source) from rope.contrib.autoimport.parse import get_names -from rope.contrib.autoimport.utils import ( - get_files, - get_modname_from_path, - get_package_tuple, - sort_and_deduplicate, - sort_and_deduplicate_tuple, -) +from rope.contrib.autoimport.utils import (get_files, get_modname_from_path, + get_package_tuple, + sort_and_deduplicate, + sort_and_deduplicate_tuple) from rope.refactor import importutils def get_future_names( - packages: List[Package], underlined: bool + packages: List[Package], underlined: bool, job_set: taskhandle.JobSet ) -> Generator[Future[Iterable[Name]], None, None]: """Get all names as futures.""" with ProcessPoolExecutor() as executor: for package in packages: for module in get_files(package, underlined): + job_set.started_job(module.modname) yield executor.submit(get_names, module, package) @@ -247,7 +246,7 @@ def generate_cache( def generate_modules_cache( self, modules: List[str] = None, - task_handle=None, + task_handle=taskhandle.NullTaskHandle(), single_thread: bool = False, underlined: bool = False, ): @@ -273,14 +272,20 @@ def generate_modules_cache( packages.append(package) packages = list(filter_packages(packages, underlined, existing)) self._add_packages(packages) + job_set = task_handle.create_jobset("Generating autoimport cache") if single_thread: for package in packages: for module in get_files(package, underlined): + job_set.started_job(module.modname) for name in get_names(module, package): self._add_name(name) + job_set.finished_job() else: - for future_name in as_completed(get_future_names(packages, underlined)): + for future_name in as_completed( + get_future_names(packages, underlined, job_set) + ): self._add_names(future_name.result()) + job_set.finished_job() self.connection.commit() diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 24291f3bc..63dab52ec 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -41,12 +41,12 @@ def get_names_from_file( underlined: bool = False, ) -> Generator[PartialName, None, None]: """Get all the names from a given file using ast.""" - with open(module, mode="rb") as file: - try: - root_node = ast.parse(file.read()) - except SyntaxError as error: - print(error) - return + + try: + root_node = ast.parse(module.read_bytes()) + except SyntaxError as error: + print(error) + return for node in ast.iter_child_nodes(root_node): if isinstance(node, ast.Assign): for target in node.targets: @@ -112,7 +112,7 @@ def get_names_from_compiled( banned = ["builtins", "python_crun"] if package in banned or (package.startswith("_") and not underlined): return # Builtins is redundant since you don't have to import it. - if source not in (Source.BUILTIN, Source.STANDARD, Source.PROJECT): + if source not in (Source.BUILTIN, Source.STANDARD): return try: module = import_module(str(package)) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index d73180988..73234d83d 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -41,9 +41,9 @@ def get_package_source( package: pathlib.Path, project: Optional[Project] = None ) -> Source: """Detect the source of a given package. Rudimentary implementation.""" - if project is not None and package.as_posix().__contains__(project.address): + if project is not None and project.address in str(package): return Source.PROJECT - if package.as_posix().__contains__("site-packages"): + if "site-packages" in package.parts: return Source.SITE_PACKAGE if package.as_posix().startswith(sys.prefix): return Source.STANDARD @@ -74,10 +74,8 @@ def get_modname_from_path( def sort_and_deduplicate(results: List[Tuple[str, int]]) -> List[str]: """Sort and deduplicate a list of name, source entries.""" - if len(results) == 0: - return [] - results.sort(key=lambda y: y[-1]) - results_sorted = list(zip(*results))[0] + results = sorted(results, key=lambda y: y[-1]) + results_sorted = [name for name, source in results] return list(OrderedDict.fromkeys(results_sorted)) @@ -85,12 +83,8 @@ def sort_and_deduplicate_tuple( results: List[Tuple[str, str, int]] ) -> List[Tuple[str, str]]: """Sort and deduplicate a list of name, module, source entries.""" - if len(results) == 0: - return [] - results.sort(key=lambda y: y[-1]) - results_sorted = [] - for result in results: - results_sorted.append(result[:-1]) + results = sorted(results, key=lambda y: y[-1]) + results_sorted = [result[:-1] for result in results] return list(OrderedDict.fromkeys(results_sorted)) @@ -108,7 +102,7 @@ def get_files( ) -> Generator[ModuleInfo, None, None]: """Find all files to parse in a given path using __init__.py.""" if package.type in (PackageType.COMPILED, PackageType.BUILTIN): - if package.source in (Source.PROJECT, Source.STANDARD, Source.BUILTIN): + if package.source in (Source.STANDARD, Source.BUILTIN): yield ModuleCompiled(None, package.name, underlined, process_imports=True) elif package.type == PackageType.SINGLE_FILE: assert package.path diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 76626ae1b..602ffb100 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -125,9 +125,7 @@ def test_search_submodule(self): def test_search_module(self): self.importer.update_module("os") import_statement = ("import os", "os") - self.assertIn( - import_statement , self.importer.search("os", exact_match=True) - ) + self.assertIn(import_statement, self.importer.search("os", exact_match=True)) self.assertIn(import_statement, self.importer.search("os")) self.assertIn(import_statement, self.importer.search("o")) From fc8d3d79e00b01a21e0fd9e7843c015fdcba66b0 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 14 Apr 2022 16:24:01 -0500 Subject: [PATCH 54/62] hack to make the job_set work --- rope/contrib/autoimport/autoimport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index ec4c0b6e8..8ff89e000 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -29,6 +29,7 @@ def get_future_names( for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) + job_set.count += 1 yield executor.submit(get_names, module, package) @@ -272,7 +273,7 @@ def generate_modules_cache( packages.append(package) packages = list(filter_packages(packages, underlined, existing)) self._add_packages(packages) - job_set = task_handle.create_jobset("Generating autoimport cache") + job_set = task_handle.create_jobset("Generating autoimport cache", 0) if single_thread: for package in packages: for module in get_files(package, underlined): From f52695bb32fcf5d19d49abef2b0336741740b0ff Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 14 Apr 2022 16:27:08 -0500 Subject: [PATCH 55/62] fix: don't increment on null job sets --- rope/contrib/autoimport/autoimport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 8ff89e000..d04e4a393 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -29,7 +29,8 @@ def get_future_names( for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) - job_set.count += 1 + if not isinstance(job_set, taskhandle.NullJobSet): + job_set.count += 1 yield executor.submit(get_names, module, package) From af14440adea7681f24accd9dcc920724e6fcbfb5 Mon Sep 17 00:00:00 2001 From: bageljr Date: Thu, 14 Apr 2022 16:56:36 -0500 Subject: [PATCH 56/62] process imports on __init__.py --- rope/contrib/autoimport/autoimport.py | 21 ++++++++++++++------- rope/contrib/autoimport/defs.py | 6 +++--- rope/contrib/autoimport/parse.py | 23 +++++++++++++++++------ rope/contrib/autoimport/utils.py | 10 ++++------ 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index d04e4a393..32a8969de 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -11,13 +11,15 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource -from rope.contrib.autoimport.defs import (ModuleFile, Name, Package, - PackageType, Source) +from rope.contrib.autoimport.defs import ModuleFile, Name, Package, PackageType, Source from rope.contrib.autoimport.parse import get_names -from rope.contrib.autoimport.utils import (get_files, get_modname_from_path, - get_package_tuple, - sort_and_deduplicate, - sort_and_deduplicate_tuple) +from rope.contrib.autoimport.utils import ( + get_files, + get_modname_from_path, + get_package_tuple, + sort_and_deduplicate, + sort_and_deduplicate_tuple, +) from rope.refactor import importutils @@ -368,7 +370,12 @@ def update_resource( resource_modname: str = get_modname_from_path( resource_path, package.path, add_package_name=False ) - module = ModuleFile(resource_path, resource_modname, underlined) + module = ModuleFile( + resource_path, + resource_modname, + underlined, + resource_path.name == "__init__.py", + ) self._del_if_exist(module_name=resource_modname, commit=False) for name in get_names(module, package): self._add_name(name) diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 51e31df2d..48ab2ee5e 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -23,7 +23,7 @@ class ModuleInfo(NamedTuple): filepath: Optional[pathlib.Path] modname: str underlined: bool - process_imports: bool = False + process_imports: bool class ModuleFile(ModuleInfo): @@ -32,7 +32,7 @@ class ModuleFile(ModuleInfo): filepath: pathlib.Path modname: str underlined: bool - process_imports: bool = False + process_imports: bool class ModuleCompiled(ModuleInfo): @@ -41,7 +41,7 @@ class ModuleCompiled(ModuleInfo): filepath = None modname: str underlined: bool - process_imports: bool = False + process_imports: bool class PackageType(Enum): diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 63dab52ec..19f1e9e80 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -33,15 +33,13 @@ def get_type_ast(node: ast.AST) -> NameType: return NameType.Function if isinstance(node, ast.Assign): return NameType.Variable - return NameType.Text # default value + return NameType.Variable # default value def get_names_from_file( - module: pathlib.Path, - underlined: bool = False, + module: pathlib.Path, underlined: bool = False, process_imports: bool = False ) -> Generator[PartialName, None, None]: """Get all the names from a given file using ast.""" - try: root_node = ast.parse(module.read_bytes()) except SyntaxError as error: @@ -66,6 +64,17 @@ def get_names_from_file( node.name, get_type_ast(node), ) + elif process_imports and isinstance(node, (ast.Import, ast.ImportFrom)): + for name in node.names: + if isinstance(name, ast.alias): + if name.asname: + real_name = name.asname + else: + real_name = name.name + else: + real_name = name + if underlined or not real_name.startswith("_"): + yield PartialName(real_name, get_type_ast(node)) def get_type_object(imported_object) -> NameType: @@ -74,7 +83,7 @@ def get_type_object(imported_object) -> NameType: return NameType.Class if inspect.isfunction(imported_object) or inspect.isbuiltin(imported_object): return NameType.Function - return NameType.Constant + return NameType.Variable def get_names(module: ModuleInfo, package: Package) -> List[Name]: @@ -86,7 +95,9 @@ def get_names(module: ModuleInfo, package: Package) -> List[Name]: if isinstance(module, ModuleFile): return [ combine(package, module, partial_name) - for partial_name in get_names_from_file(module.filepath, module.underlined) + for partial_name in get_names_from_file( + module.filepath, module.underlined, module.process_imports + ) ] return [] diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index 73234d83d..bb49b6a49 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -103,11 +103,11 @@ def get_files( """Find all files to parse in a given path using __init__.py.""" if package.type in (PackageType.COMPILED, PackageType.BUILTIN): if package.source in (Source.STANDARD, Source.BUILTIN): - yield ModuleCompiled(None, package.name, underlined, process_imports=True) + yield ModuleCompiled(None, package.name, underlined, True) elif package.type == PackageType.SINGLE_FILE: assert package.path assert package.path.suffix == ".py" - yield ModuleFile(package.path, package.path.stem, underlined) + yield ModuleFile(package.path, package.path.stem, underlined, False) else: assert package.path for file in package.path.glob("*.py"): @@ -116,11 +116,9 @@ def get_files( file, get_modname_from_path(file.parent, package.path), underlined, - process_imports=True, + True, ) elif should_parse(file, underlined): yield ModuleFile( - file, - get_modname_from_path(file, package.path), - underlined, + file, get_modname_from_path(file, package.path), underlined, False ) From 77a9e5f749b9c19e4f1d3c9da9a6ee4b6f8cd717 Mon Sep 17 00:00:00 2001 From: bageljr Date: Sat, 16 Apr 2022 21:16:25 -0500 Subject: [PATCH 57/62] split out search --- rope/contrib/autoimport/autoimport.py | 71 ++++++++++++++------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 32a8969de..37e6f1d3a 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -11,7 +11,14 @@ from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource -from rope.contrib.autoimport.defs import ModuleFile, Name, Package, PackageType, Source +from rope.contrib.autoimport.defs import ( + ModuleFile, + Name, + NameType, + Package, + PackageType, + Source, +) from rope.contrib.autoimport.parse import get_names from rope.contrib.autoimport.utils import ( get_files, @@ -136,48 +143,43 @@ def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: if not exact_match: name = name + "%" # Makes the query a starts_with query results: List[Tuple[str, str, int]] = [] - for import_name, module, source in self.connection.execute( - "SELECT name, module, source FROM names WHERE name LIKE (?)", (name,) - ): - results.append((f"from {module} import {import_name}", import_name, source)) - for module, source in self.connection.execute( - "Select module, source FROM names where module LIKE (?)", ("%." + name,) - ): - parts = module.split(".") - import_name = parts[-1] - remaining = parts[0] - for part in parts[1:-1]: - remaining += "." - remaining += part - results.append( - (f"from {remaining} import {import_name}", import_name, source) - ) - for module, source in self.connection.execute( - "Select module, source from names where module LIKE (?)", (name,) + for statement, import_name, source, type in self.search_name(name, exact_match): + results.append((statement, import_name, source)) + for statement, import_name, source, type in self.search_module( + name, exact_match ): - results.append((f"import {module}", module, source)) + results.append((statement, import_name, source)) return sort_and_deduplicate_tuple(results) - def lsp_search( + def search_name( self, name: str, exact_match: bool = False - ) -> Tuple[List[Tuple[str, str, int, int]], List[Tuple[str, str, int, int]]]: + ) -> Generator[Tuple[str, str, int, int], None, None]: """ - Search both modules and names for an import string. + Search both names for avalible imports. - Returns the name, import statement, source, split into normal names and modules. + Returns the import statement, import name, source, and type. """ if not exact_match: name = name + "%" # Makes the query a starts_with query - results_name: List[Tuple[str, str, int, int]] = [] - results_module: List[Tuple[str, str, int, int]] = [] for import_name, module, source, name_type in self.connection.execute( "SELECT name, module, source, type FROM names WHERE name LIKE (?)", (name,) ): - results_name.append( + yield ( (f"from {module} import {import_name}", import_name, source, name_type) ) - for module, source, name_type in self.connection.execute( - "Select module, source, type FROM names where module LIKE (?)", + + def search_module( + self, name: str, exact_match: bool = False + ) -> Generator[Tuple[str, str, int, int], None, None]: + """ + Search both modules for avalible imports. + + Returns the import statement, import name, source, and type. + """ + if not exact_match: + name = name + "%" # Makes the query a starts_with query + for module, source in self.connection.execute( + "Select module, source FROM names where module LIKE (?)", ("%." + name,), ): parts = module.split(".") @@ -186,19 +188,18 @@ def lsp_search( for part in parts[1:-1]: remaining += "." remaining += part - results_module.append( + yield ( ( f"from {remaining} import {import_name}", import_name, source, - name_type, + NameType.Module.value, ) ) - for module, source, name_type in self.connection.execute( - "Select module, source, type from names where module LIKE (?)", (name,) + for module, source in self.connection.execute( + "Select module, source from names where module LIKE (?)", (name,) ): - results_module.append((f"import {module}", module, source, name_type)) - return results_name, results_module + yield ((f"import {module}", module, source, NameType.Module.value)) def get_modules(self, name) -> List[str]: """Get the list of modules that have global `name`.""" From 15b82ad347c11092cc8f6ca2aca31c484c7c31ef Mon Sep 17 00:00:00 2001 From: bageljr Date: Sat, 16 Apr 2022 21:42:20 -0500 Subject: [PATCH 58/62] ignore submodules for vanilla import statements --- rope/contrib/autoimport/autoimport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 37e6f1d3a..1538d09e7 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -199,6 +199,8 @@ def search_module( for module, source in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): + if '.' in module: + continue yield ((f"import {module}", module, source, NameType.Module.value)) def get_modules(self, name) -> List[str]: From 1bac1553e88ba054f28fb971132b88a9e5e3ff86 Mon Sep 17 00:00:00 2001 From: bageljr Date: Fri, 22 Apr 2022 12:40:15 -0500 Subject: [PATCH 59/62] implement search_full with the ability to ignore names already present in the file --- rope/contrib/autoimport/autoimport.py | 85 ++++++++++++++++++++------- rope/contrib/autoimport/defs.py | 9 +++ 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 1538d09e7..4174bfd97 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -6,7 +6,7 @@ from collections import OrderedDict from concurrent.futures import Future, ProcessPoolExecutor, as_completed from itertools import chain -from typing import Generator, Iterable, List, Optional, Tuple +from typing import Generator, Iterable, List, Optional, Set, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project @@ -17,9 +17,10 @@ NameType, Package, PackageType, + SearchResult, Source, ) -from rope.contrib.autoimport.parse import get_names +from rope.contrib.autoimport.parse import get_names, get_names_from_file from rope.contrib.autoimport.utils import ( get_files, get_modname_from_path, @@ -114,6 +115,7 @@ def _setup_db(self): def import_assist(self, starting: str): """ Find modules that have a global name that starts with `starting`. + For a more complete list including modules, use the search or search_full methods. Parameters __________ @@ -137,23 +139,59 @@ def import_assist(self, starting: str): def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: """ Search both modules and names for an import string. + This is a simple wrapper around search_full with basic sorting based on Source. - Returns list of import statement ,modname pairs + Returns a sorted list of import statement, modname pairs """ - if not exact_match: - name = name + "%" # Makes the query a starts_with query - results: List[Tuple[str, str, int]] = [] - for statement, import_name, source, type in self.search_name(name, exact_match): - results.append((statement, import_name, source)) - for statement, import_name, source, type in self.search_module( - name, exact_match - ): - results.append((statement, import_name, source)) + results: List[Tuple[str, str, int]] = [ + (statement, import_name, source) + for statement, import_name, source, type in self.search_full( + name, exact_match + ) + ] return sort_and_deduplicate_tuple(results) - def search_name( + def search_full( + self, + name: str, + exact_match: bool = False, + resource: Optional[Resource] = None, + ignored_names: Set[str] = set(), + ) -> Generator[SearchResult, None, None]: + """ + Search both modules and names for an import string. + + Parameters + __________ + name: str + Name to search for + exact_match: bool + If using exact_match, only search for that name. + Otherwise, search for any name starting with that name. + resource_name : Optional[Resource] + Will ignore any names from this resource. + Since it uses Ast, and reads from the saved file, + its reccomennded to get the names from the client + ignored_names : Set[str] + Will ignore any names in this set + + Return + __________ + Unsorted Generator of SearchResults. Each is guaranteed to be unique. + """ + if resource: + resource_path: pathlib.Path = pathlib.Path(resource.real_path) + for exisiting_name, type in get_names_from_file(resource_path, True, True): + ignored_names.add(exisiting_name) + results = set(self._search_name(name, exact_match)) + results = results.union(self._search_module(name, exact_match)) + for result in results: + if result.name not in ignored_names: + yield result + + def _search_name( self, name: str, exact_match: bool = False - ) -> Generator[Tuple[str, str, int, int], None, None]: + ) -> Generator[SearchResult, None, None]: """ Search both names for avalible imports. @@ -165,12 +203,17 @@ def search_name( "SELECT name, module, source, type FROM names WHERE name LIKE (?)", (name,) ): yield ( - (f"from {module} import {import_name}", import_name, source, name_type) + SearchResult( + f"from {module} import {import_name}", + import_name, + source, + name_type, + ) ) - def search_module( + def _search_module( self, name: str, exact_match: bool = False - ) -> Generator[Tuple[str, str, int, int], None, None]: + ) -> Generator[SearchResult, None, None]: """ Search both modules for avalible imports. @@ -189,7 +232,7 @@ def search_module( remaining += "." remaining += part yield ( - ( + SearchResult( f"from {remaining} import {import_name}", import_name, source, @@ -199,9 +242,11 @@ def search_module( for module, source in self.connection.execute( "Select module, source from names where module LIKE (?)", (name,) ): - if '.' in module: + if "." in module: continue - yield ((f"import {module}", module, source, NameType.Module.value)) + yield SearchResult( + f"import {module}", module, source, NameType.Module.value + ) def get_modules(self, name) -> List[str]: """Get the list of modules that have global `name`.""" diff --git a/rope/contrib/autoimport/defs.py b/rope/contrib/autoimport/defs.py index 48ab2ee5e..b5559221c 100644 --- a/rope/contrib/autoimport/defs.py +++ b/rope/contrib/autoimport/defs.py @@ -107,3 +107,12 @@ class PartialName(NamedTuple): name: str name_type: NameType + + +class SearchResult(NamedTuple): + """Search Result.""" + + import_statement: str + name: str + source: int + itemkind: int From b952b205632e786ddb377596f468cd6ab85cce02 Mon Sep 17 00:00:00 2001 From: bageljr Date: Fri, 22 Apr 2022 15:25:53 -0500 Subject: [PATCH 60/62] use CREATE INDEX,remove unnessecary checks --- rope/contrib/autoimport/autoimport.py | 34 +++++---------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 4174bfd97..9e457f22e 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -96,7 +96,6 @@ def __init__(self, project, observe=True, underlined=False, memory=True): db_path = f"{project.ropefolder.path}/autoimport.db" self.connection = sqlite3.connect(db_path) self._setup_db() - self._check_all() if observe: observer = resourceobserver.ResourceObserver( changed=self._changed, moved=self._moved, removed=self._removed @@ -110,12 +109,16 @@ def _setup_db(self): ) self.connection.execute(f"create table if not exists names{names_table}") self.connection.execute(f"create table if not exists packages{packages_table}") + self.connection.execute("CREATE INDEX IF NOT EXISTS name on names(name)") + self.connection.execute("CREATE INDEX IF NOT EXISTS module on names(module)") + self.connection.execute("CREATE INDEX IF NOT EXISTS package on names(package)") self.connection.commit() def import_assist(self, starting: str): """ Find modules that have a global name that starts with `starting`. - For a more complete list including modules, use the search or search_full methods. + + For a more complete list, use the search or search_full methods. Parameters __________ @@ -129,9 +132,6 @@ def import_assist(self, starting: str): "select name, module, source from names WHERE name LIKE (?)", (starting + "%",), ).fetchall() - for result in results: - if not self._check_import(result[1]): - del results[result] return sort_and_deduplicate_tuple( results ) # Remove duplicates from multiple occurences of the same item @@ -139,6 +139,7 @@ def import_assist(self, starting: str): def search(self, name: str, exact_match: bool = False) -> List[Tuple[str, str]]: """ Search both modules and names for an import string. + This is a simple wrapper around search_full with basic sorting based on Source. Returns a sorted list of import statement, modname pairs @@ -253,20 +254,15 @@ def get_modules(self, name) -> List[str]: results = self.connection.execute( "SELECT module, source FROM names WHERE name LIKE (?)", (name,) ).fetchall() - for result in results: - if not self._check_import(result[0]): - del results[result] return sort_and_deduplicate(results) def get_all_names(self) -> List[str]: """Get the list of all cached global names.""" - self._check_all() results = self.connection.execute("select name from names").fetchall() return results def _dump_all(self) -> Tuple[List[Name], List[Package]]: """Dump the entire database.""" - self._check_all() name_results = self.connection.execute("select * from names").fetchall() package_results = self.connection.execute("select * from packages").fetchall() return name_results, package_results @@ -520,24 +516,6 @@ def _add_name(self, name: Name): ), ) - def _check_import(self, module: pathlib.Path) -> bool: - """ - Check the ability to import an external package, removes it if not avalible. - - Parameters - ---------- - module: pathlib.path - The module to check - Returns - ---------- - """ - # Not Implemented Yet, silently will fail - return True - - def _check_all(self): - """Check all modules and removes bad ones.""" - pass - def _find_package_path(self, target_name: str) -> Optional[Package]: if target_name in sys.builtin_module_names: return Package(target_name, Source.BUILTIN, None, PackageType.BUILTIN) From 49bc66e1f3031c964f6ed7ce0d07e7533a1fbde8 Mon Sep 17 00:00:00 2001 From: bageljr Date: Fri, 22 Apr 2022 16:13:10 -0500 Subject: [PATCH 61/62] use a multithread executor to dramatically speed up indexing of local files --- rope/contrib/autoimport/autoimport.py | 110 +++++++++++++------------- 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 9e457f22e..2173541f7 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -71,9 +71,10 @@ class AutoImport: connection: sqlite3.Connection underlined: bool - project: Project + rope_project: Project + project: Package - def __init__(self, project, observe=True, underlined=False, memory=True): + def __init__(self, project: Project, observe=True, underlined=False, memory=True): """Construct an AutoImport object. Parameters @@ -87,7 +88,13 @@ def __init__(self, project, observe=True, underlined=False, memory=True): memory : bool if true, don't persist to disk """ - self.project = project + self.rope_project = project + project_package = get_package_tuple( + pathlib.Path(project.root.real_path), project + ) + assert project_package is not None + assert project_package.path is not None + self.project = project_package self.underlined = underlined db_path: str if memory or project.ropefolder is None: @@ -280,14 +287,21 @@ def generate_cache( project are cached. """ if resources is None: - resources = self.project.get_python_files() + resources = self.rope_project.get_python_files() job_set = task_handle.create_jobset( "Generating autoimport cache", len(resources) ) - # Should be very fast, so doesn't need multithreaded computation - for file in resources: - job_set.started_job(f"Working on {file.path}") - self.update_resource(file, underlined, commit=False) + self.connection.execute( + "delete from names where package = ?", (self.project.name,) + ) + futures = [] + with ProcessPoolExecutor() as executor: + for file in resources: + job_set.started_job(f"Working on {file.path}") + module = self._resource_to_module(file, underlined) + futures.append(executor.submit(get_names, module, self.project)) + for future in as_completed(futures): + self._add_names(future.result()) job_set.finished_job() self.connection.commit() @@ -356,9 +370,9 @@ def get_name_locations(self, name): for module in modules: try: module_name = module[0] - if module_name.startswith(f"{self._project_name}."): + if module_name.startswith(f"{self.project.name}."): module_name = ".".join(module_name.split(".")) - pymodule = self.project.get_module(module_name) + pymodule = self.rope_project.get_module(module_name) if name in pymodule: pyname = pymodule[name] module, lineno = pyname.get_definition_location() @@ -387,12 +401,12 @@ def find_insertion_line(self, code): if match is not None: code = code[: match.start()] try: - pymodule = libutils.get_string_module(self.project, code) + pymodule = libutils.get_string_module(self.rope_project, code) except exceptions.ModuleSyntaxError: return 1 testmodname = "__rope_testmodule_rope" importinfo = importutils.NormalImport(((testmodname, None),)) - module_imports = importutils.get_module_imports(self.project, pymodule) + module_imports = importutils.get_module_imports(self.rope_project, pymodule) module_imports.add_import(importinfo) code = module_imports.get_changed_source() offset = code.index(testmodname) @@ -404,24 +418,9 @@ def update_resource( ): """Update the cache for global names in `resource`.""" underlined = underlined if underlined else self.underlined - package = get_package_tuple(self._project_path, self.project) - if package is None or package.path is None: - return - resource_path: pathlib.Path = pathlib.Path(resource.real_path) - # The project doesn't need its name added to the path, - # since the standard python file layout accounts for that - # so we set add_package_name to False - resource_modname: str = get_modname_from_path( - resource_path, package.path, add_package_name=False - ) - module = ModuleFile( - resource_path, - resource_modname, - underlined, - resource_path.name == "__init__.py", - ) - self._del_if_exist(module_name=resource_modname, commit=False) - for name in get_names(module, package): + module = self._resource_to_module(resource, underlined) + self._del_if_exist(module_name=module.modname, commit=False) + for name in get_names(module, self.project): self._add_name(name) if commit: self.connection.commit() @@ -432,7 +431,7 @@ def _changed(self, resource): def _moved(self, resource: Resource, newresource: Resource): if not resource.is_folder(): - modname = self._modname(resource) + modname = self._resource_to_module(resource).modname self._del_if_exist(modname) self.update_resource(newresource) @@ -442,7 +441,7 @@ def _del_if_exist(self, module_name, commit: bool = True): self.connection.commit() def _get_python_folders(self) -> List[pathlib.Path]: - folders = self.project.get_python_path_folders() + folders = self.rope_project.get_python_path_folders() folder_paths = [ pathlib.Path(folder.path) for folder in folders if folder.path != "/usr/bin" ] @@ -455,7 +454,7 @@ def _get_available_packages(self) -> List[Package]: ] for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_tuple(package, self.project) + package_tuple = get_package_tuple(package, self.rope_project) if package_tuple is None: continue packages.append(package_tuple) @@ -469,32 +468,12 @@ def _get_existing(self) -> List[str]: existing: List[str] = list( chain(*self.connection.execute("select * from packages").fetchall()) ) - existing.append(self._project_name) + existing.append(self.project.name) return existing - @property - def _project_name(self): - package_path: pathlib.Path = pathlib.Path(self.project.address) - package_tuple = get_package_tuple(package_path) - if package_tuple is None: - return None - return package_tuple[0] - - @property - def _project_path(self): - return pathlib.Path(self.project.address) - - def _modname(self, resource: Resource): - resource_path: pathlib.Path = pathlib.Path(resource.real_path) - package_path: pathlib.Path = pathlib.Path(self.project.address) - resource_modname: str = get_modname_from_path( - resource_path, package_path, add_package_name=False - ) - return resource_modname - def _removed(self, resource): if not resource.is_folder(): - modname = self._modname(resource) + modname = self._resource_to_module(resource).modname self._del_if_exist(modname) def _add_future_names(self, names: Future[List[Name]]): @@ -521,7 +500,7 @@ def _find_package_path(self, target_name: str) -> Optional[Package]: return Package(target_name, Source.BUILTIN, None, PackageType.BUILTIN) for folder in self._get_python_folders(): for package in folder.iterdir(): - package_tuple = get_package_tuple(package, self.project) + package_tuple = get_package_tuple(package, self.rope_project) if package_tuple is None: continue name, source, package_path, package_type = package_tuple @@ -529,3 +508,22 @@ def _find_package_path(self, target_name: str) -> Optional[Package]: return package_tuple return None + + def _resource_to_module( + self, resource: Resource, underlined: bool = False + ) -> ModuleFile: + assert self.project.path + underlined = underlined if underlined else self.underlined + resource_path: pathlib.Path = pathlib.Path(resource.real_path) + # The project doesn't need its name added to the path, + # since the standard python file layout accounts for that + # so we set add_package_name to False + resource_modname: str = get_modname_from_path( + resource_path, self.project.path, add_package_name=False + ) + return ModuleFile( + resource_path, + resource_modname, + underlined, + resource_path.name == "__init__.py", + ) From 4a99a77f84a1fa9438399608c332ee87047eb8da Mon Sep 17 00:00:00 2001 From: bageljr Date: Wed, 27 Apr 2022 11:19:52 -0500 Subject: [PATCH 62/62] ignore imports which don't belong to the package --- rope/contrib/autoimport/autoimport.py | 17 ++++++----------- rope/contrib/autoimport/parse.py | 17 ++++++++++++++--- ropetest/contrib/autoimport/conftest.py | 3 --- ropetest/contrib/autoimport/parsetest.py | 4 ++++ ropetest/contrib/autoimporttest.py | 11 +++++++++++ 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/rope/contrib/autoimport/autoimport.py b/rope/contrib/autoimport/autoimport.py index 2173541f7..7e03d38d9 100644 --- a/rope/contrib/autoimport/autoimport.py +++ b/rope/contrib/autoimport/autoimport.py @@ -20,7 +20,7 @@ SearchResult, Source, ) -from rope.contrib.autoimport.parse import get_names, get_names_from_file +from rope.contrib.autoimport.parse import get_names from rope.contrib.autoimport.utils import ( get_files, get_modname_from_path, @@ -32,7 +32,9 @@ def get_future_names( - packages: List[Package], underlined: bool, job_set: taskhandle.JobSet + packages: List[Package], + underlined: bool, + job_set: taskhandle.JobSet, ) -> Generator[Future[Iterable[Name]], None, None]: """Get all names as futures.""" with ProcessPoolExecutor() as executor: @@ -163,7 +165,6 @@ def search_full( self, name: str, exact_match: bool = False, - resource: Optional[Resource] = None, ignored_names: Set[str] = set(), ) -> Generator[SearchResult, None, None]: """ @@ -176,10 +177,6 @@ def search_full( exact_match: bool If using exact_match, only search for that name. Otherwise, search for any name starting with that name. - resource_name : Optional[Resource] - Will ignore any names from this resource. - Since it uses Ast, and reads from the saved file, - its reccomennded to get the names from the client ignored_names : Set[str] Will ignore any names in this set @@ -187,10 +184,6 @@ def search_full( __________ Unsorted Generator of SearchResults. Each is guaranteed to be unique. """ - if resource: - resource_path: pathlib.Path = pathlib.Path(resource.real_path) - for exisiting_name, type in get_names_from_file(resource_path, True, True): - ignored_names.add(exisiting_name) results = set(self._search_name(name, exact_match)) results = results.union(self._search_module(name, exact_match)) for result in results: @@ -333,6 +326,8 @@ def generate_modules_cache( continue packages.append(package) packages = list(filter_packages(packages, underlined, existing)) + if len(packages) == 0: + return self._add_packages(packages) job_set = task_handle.create_jobset("Generating autoimport cache", 0) if single_thread: diff --git a/rope/contrib/autoimport/parse.py b/rope/contrib/autoimport/parse.py index 19f1e9e80..433e54bf5 100644 --- a/rope/contrib/autoimport/parse.py +++ b/rope/contrib/autoimport/parse.py @@ -37,7 +37,10 @@ def get_type_ast(node: ast.AST) -> NameType: def get_names_from_file( - module: pathlib.Path, underlined: bool = False, process_imports: bool = False + module: pathlib.Path, + package_name: str = "", + underlined: bool = False, + process_imports: bool = False, ) -> Generator[PartialName, None, None]: """Get all the names from a given file using ast.""" try: @@ -64,7 +67,12 @@ def get_names_from_file( node.name, get_type_ast(node), ) - elif process_imports and isinstance(node, (ast.Import, ast.ImportFrom)): + elif process_imports and isinstance(node, ast.ImportFrom): + # When we process imports, we want to include names in it's own package. + if node.level == 0: + continue + if not node.module or package_name is node.module.split(".")[0]: + continue for name in node.names: if isinstance(name, ast.alias): if name.asname: @@ -96,7 +104,10 @@ def get_names(module: ModuleInfo, package: Package) -> List[Name]: return [ combine(package, module, partial_name) for partial_name in get_names_from_file( - module.filepath, module.underlined, module.process_imports + module.filepath, + package.name, + underlined=module.underlined, + process_imports=module.process_imports, ) ] return [] diff --git a/ropetest/contrib/autoimport/conftest.py b/ropetest/contrib/autoimport/conftest.py index 7e7044032..c539eb325 100644 --- a/ropetest/contrib/autoimport/conftest.py +++ b/ropetest/contrib/autoimport/conftest.py @@ -34,9 +34,6 @@ def typing_path(): yield pathlib.Path(typing.__file__) - - - @pytest.fixture def build_env_path(): from build import env diff --git a/ropetest/contrib/autoimport/parsetest.py b/ropetest/contrib/autoimport/parsetest.py index e37db9cee..a5b82e699 100644 --- a/ropetest/contrib/autoimport/parsetest.py +++ b/ropetest/contrib/autoimport/parsetest.py @@ -5,6 +5,10 @@ def test_typing_names(typing_path): names = list(parse.get_names_from_file(typing_path)) assert PartialName("Dict", NameType.Variable) in names + import typing + name_set = set(name.name for name in names) + for name in typing.__all__: + assert name in name_set def test_find_sys(): diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 602ffb100..5d7b2eef2 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -138,6 +138,17 @@ def test_search(self): self.assertIn(import_statement, self.importer.search("Di")) self.assertIn(import_statement, self.importer.search("D")) + def test_typing_all(self): + import typing + + self.importer._del_if_exist("typing") + self.importer.generate_modules_cache(["typing"], single_thread=True) + for item in typing.__all__: + self.assertIn( + (f"from typing import {item}", item), + self.importer.search(item, exact_match=True), + ) + def test_generate_full_cache(self): """The single thread test takes much longer than the multithread test but is easier to debug""" single_thread = False