From 5280d3428c3ced7315db1a074316334db2ed5f6f Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 16 Sep 2019 12:57:44 -0700 Subject: [PATCH] Merge persistence work into master (#1482) * Persistent analysis, part I (#1224) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * Merge master into database branch (#1312) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * Avoid empty node in empty global scope (#1226) * Add null checks * Avoid null node in empty global scope * Remove unrelated change * Lock content since it is locked in other cases (#1227) * Fix addBrackets setting (#1241) * Fix addBrackets * Remove unrelated change * Usings * Handle library analysis better (#1246) * Trying to augment argument set with Evaluator and Expression context (#1259) * Moving anyStr to its own method to prevent FromTypeVar from being called with an ArgumentSet that has no context * Trying to augment argument set with context wherever possible * Setting expression in argument set, removing null * Renaming to WithoutContext and updating comment * Check for null args in PythonFunctionOverload * When returning in init method, report a diagnostic error (#1261) * When returning in init method, report a diagnostic error * Adding check and tests for returning None before adding error for a return statement in the init function * Cleaning up tests * Give Diagnostic messages on improper usage of Generic (#1248) * Adding diagnostic message for improper use Generic. * Adding tests to make sure no diagnostics on valid uses of Generic * Adding diagnostic error on binary operations with incompatible types (#1254) * Adding diagnostic error for binary operations with incompatible types - e.g 5 + 'str' * When NewType is called and the first argument is not a string, make a diagnostic message (#1260) * Adding diagnostic error if the user calls NewType with the first arg not of string type * Fix typo in TROUBLESHOOTING.md (#1285) * Multiple analysis fixes (#1297) Multiple analysis fixes. - Fixes #1151: Reliable way to determine final analysis of a module - Fixes #1201: NRE in SymbolCollector.AddProperty - Fixes #1228: NRE in TryFindMissingDependencies - Fixes #1294: Handle valid case for reloading AST - Fixes #1295: FindReferences doesn't work on package init members - Fixes #1296: DependencyResolver graph misses intermediate imports - Part of the fix for #1174: All references are not listed when using ms language server (fixes in analysis itself are also required for this one) * Rework search path resolution (#1289) * first working path classfier, breaks existing path code for now * inherit Comparer instead of implementing both * don't do path depth comparison, preserve user ordering as before * switch to using new path classification code in main module resolution * fix typo * clean up comment about paths in Server.cs * use String.Split instead of regex when parsing script output * test ordering of user provided paths * add new normalize and trim helper, check preconditions as debug asserts rather than commenting on them * use Split extension, don't MaybeEnumerate paths * No diagnostic message when Generic is called with no args (#1305) * No diagnostic message when Generic is called with no args * When a class inherits from something that is not a class, give a diagnostic error (#1277) * When analyzing class definitions, give diagnostic if bases are not all class types * PR feedback * Treat persistent module as regular specialized (#1334) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Fix merge issue * Remove debug code * Store location of module members in the database (#1339) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Add location converter abstraction * Store member location * Basic locations test * Navigation * Add test * Update baselines * Port changes from dbtype * Multiple fixes and additional tests for persistence (#1372) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Add location converter abstraction * Store member location * Fix merge issue * Basic locations test * Navigation * Add test * Update baselines * Type restore - initial * Remove debug code * Partial * Fix stub merge * Various model fixes * Improve module handling * Qualified name improvements * Fix unbound case * Improve stub merge * Fix qualified names of typing * Handle stub-only modules * Add tests for io, re and sys * Better handle named tuple * Handle module circular references * Handle else in platform and version clauses + handle it in symbol collector * Formatting * Fix walk of multi-level if statement * Unify package search in imports * Fix tests * Undo change * Port changes from dbtype * Partial * Named tuple support * Baseline updates * Debug code * Support types * Properly compare class member declaring type * Nested classes and functions persistence * Undo debug * Fix numpy restore * Baselines * Fix tests * Update AnyStr test reflecting changes to AnyStr behavior * Exclude baselines from git diff * Fix gitattr * Move git setting to root * Try no path * Add RDT count to the analysis_complete event (#1396) * Add RDT count to the analysis_complete event * Use property instead of method * Test fixes * Undo change * Additional stub merge fixes * Baseline updates * Fix goto def * Protect specific type creation * Track documentaton source * More reliable tests + simplification * Prevent crashes when the AST happens to be null (#1405) * Remove all references to dropped modules and force GC on reload (#1402) * First attempt at ensuring moduled dropped during reload are GC'd * Make sure AST is set after content is cleared * More tests passing * ModuleWalker updates * Also don't merge against specialized modules, do builtin _astMap reset before reanalyzing instead of after any analysis * Don't call dispose directly, it's done elsewhere and has no effect * Force GC via normal session instead of waiting for a hardcoded time * Un-refactor function since it's not being called externally anymore * Formatting/usings * PR feedback * Undo IsBuiltin, some things continued to be referenced with that changed * Move reload logic down into PythonAnalyzer, remove added public inferface items * Mode AddReference to PythonType as override * Remove dead code, make all ResetAnalyzer calls full * Typo * Cleanup * Basic classification * Fix merge error * Module unique id fixes * Stricted check to save analysis * Revert "Fix tests" This reverts commit 247a8c38741324452cc5a8eeb412a9784441bd8b. * Revert "Unify package search in imports" This reverts commit 67fed10d1c892de07a9a920b5396d53d7473b977. * Don't clear scope variables with inner classes * Fix typo * Add persistence tests for 60+ system modules (#1429) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Add location converter abstraction * Store member location * Fix merge issue * Basic locations test * Navigation * Add test * Update baselines * Type restore - initial * Remove debug code * Partial * Fix stub merge * Various model fixes * Improve module handling * Qualified name improvements * Fix unbound case * Improve stub merge * Fix qualified names of typing * Handle stub-only modules * Add tests for io, re and sys * Better handle named tuple * Handle module circular references * Handle else in platform and version clauses + handle it in symbol collector * Formatting * Fix walk of multi-level if statement * Unify package search in imports * Fix tests * Undo change * Port changes from dbtype * Partial * Named tuple support * Baseline updates * Debug code * Support types * Properly compare class member declaring type * Nested classes and functions persistence * Undo debug * Fix numpy restore * Baselines * Fix tests * Update AnyStr test reflecting changes to AnyStr behavior * Exclude baselines from git diff * Fix gitattr * Move git setting to root * Try no path * Test fixes * Undo change * Additional stub merge fixes * Baseline updates * Fix goto def * Protect specific type creation * Track documentaton source * More reliable tests + simplification * Typo * Cleanup * Basic classification * Fix merge error * Module unique id fixes * Stricted check to save analysis * Revert "Fix tests" This reverts commit 247a8c38741324452cc5a8eeb412a9784441bd8b. * Revert "Unify package search in imports" This reverts commit 67fed10d1c892de07a9a920b5396d53d7473b977. * Don't clear scope variables with inner classes * Fix typo * Many new tests * Fix collections test * Fix CTypes * Initial * Update test * Merge issues * Fix CTypes again * Fix null bases * Tell between class members with/without self better * TypeVar support * Add bound/covariant * Fix Enum reassignments * Fix Random * Fix import * over local declarations (Socket) * Move interface * Fix reference search * Enable asyncio test * More tests * Enable dataclasses * Add inspect and gzip * More tests * Add setting * Add handling of import position relative to the variable * Caching level changes * Update baselines * Test update (we no longer reassign classes) * Fix typing tests * Update baseline * Fix missing keys issue (#1451) * Typevar fix * Persistence of generics and named tuples (#1459) * Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Add location converter abstraction * Store member location * Fix merge issue * Basic locations test * Navigation * Add test * Update baselines * Type restore - initial * Remove debug code * Partial * Fix stub merge * Various model fixes * Improve module handling * Qualified name improvements * Fix unbound case * Improve stub merge * Fix qualified names of typing * Handle stub-only modules * Add tests for io, re and sys * Better handle named tuple * Handle module circular references * Handle else in platform and version clauses + handle it in symbol collector * Formatting * Fix walk of multi-level if statement * Unify package search in imports * Fix tests * Undo change * Port changes from dbtype * Partial * Named tuple support * Baseline updates * Debug code * Support types * Properly compare class member declaring type * Nested classes and functions persistence * Undo debug * Fix numpy restore * Baselines * Fix tests * Update AnyStr test reflecting changes to AnyStr behavior * Exclude baselines from git diff * Fix gitattr * Move git setting to root * Try no path * Test fixes * Undo change * Additional stub merge fixes * Baseline updates * Fix goto def * Protect specific type creation * Track documentaton source * More reliable tests + simplification * Typo * Cleanup * Basic classification * Fix merge error * Module unique id fixes * Stricted check to save analysis * Revert "Fix tests" This reverts commit 247a8c38741324452cc5a8eeb412a9784441bd8b. * Revert "Unify package search in imports" This reverts commit 67fed10d1c892de07a9a920b5396d53d7473b977. * Don't clear scope variables with inner classes * Fix typo * Many new tests * Fix collections test * Fix CTypes * Initial * Update test * Merge issues * Fix CTypes again * Fix null bases * Tell between class members with/without self better * TypeVar support * Add bound/covariant * Fix Enum reassignments * Fix Random * Fix import * over local declarations (Socket) * Move interface * Fix reference search * Enable asyncio test * More tests * Enable dataclasses * Add inspect and gzip * More tests * Add setting * Add handling of import position relative to the variable * Caching level changes * Partial * Rework * Fix null * Update baselines * Functools pass * Reverse stub merge * Partial * Partial ctypes * Update variables post merge * Fix ctypes * Merge issues * Fix typevar * Add caching on construction * Assorted fixes * Named tuples * Work around os stub oddness * Fix generic qualified names * Ignore lambdas * Support named tuples as bases * Baselines * Test updates * Named tuple tests * PR feedback * Using * Move diagnostics to GenericBase * PR feedback * - Fix #1455: AF: Library module of type Stub has been analyzed already! (#1470) - Restore task handling in unit tests - Throw exception from Debug.Fail instead of FailFast * Update baselines since __spec__ member was added * Undo some of changes to match master. * Fix null ref * Resolve conflicts * Fix null ref on bases due to new AnyStr behavior * Handling dependencies in module persistence (#1471) * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback * PR feedback * Restore persistence + update test * Better handle persistent module in dependencies * Undo * Add location converter abstraction * Store member location * Fix merge issue * Basic locations test * Navigation * Add test * Update baselines * Type restore - initial * Remove debug code * Partial * Fix stub merge * Various model fixes * Improve module handling * Qualified name improvements * Fix unbound case * Improve stub merge * Fix qualified names of typing * Handle stub-only modules * Add tests for io, re and sys * Better handle named tuple * Handle module circular references * Handle else in platform and version clauses + handle it in symbol collector * Formatting * Fix walk of multi-level if statement * Unify package search in imports * Fix tests * Undo change * Port changes from dbtype * Partial * Named tuple support * Baseline updates * Debug code * Support types * Properly compare class member declaring type * Nested classes and functions persistence * Undo debug * Fix numpy restore * Baselines * Fix tests * Update AnyStr test reflecting changes to AnyStr behavior * Exclude baselines from git diff * Fix gitattr * Move git setting to root * Try no path * Test fixes * Undo change * Additional stub merge fixes * Baseline updates * Fix goto def * Protect specific type creation * Track documentaton source * More reliable tests + simplification * Typo * Cleanup * Basic classification * Fix merge error * Module unique id fixes * Stricted check to save analysis * Revert "Fix tests" This reverts commit 247a8c38741324452cc5a8eeb412a9784441bd8b. * Revert "Unify package search in imports" This reverts commit 67fed10d1c892de07a9a920b5396d53d7473b977. * Don't clear scope variables with inner classes * Fix typo * Many new tests * Fix collections test * Fix CTypes * Initial * Update test * Merge issues * Fix CTypes again * Fix null bases * Tell between class members with/without self better * TypeVar support * Add bound/covariant * Fix Enum reassignments * Fix Random * Fix import * over local declarations (Socket) * Move interface * Fix reference search * Enable asyncio test * More tests * Enable dataclasses * Add inspect and gzip * More tests * Add setting * Add handling of import position relative to the variable * Caching level changes * Partial * Rework * Fix null * Update baselines * Functools pass * Reverse stub merge * Partial * Partial ctypes * Update variables post merge * Fix ctypes * Merge issues * Fix typevar * Add caching on construction * Assorted fixes * Named tuples * Work around os stub oddness * Fix generic qualified names * Ignore lambdas * Support named tuples as bases * Baselines * Test updates * Named tuple tests * PR feedback * Using * Separate creation and population for better handling of forward references. * Add dependency provider * Merge issues * Better class creation * Partial * Dependency providers * Add IsPersistent flag * Make special global scope * Move dependency fetch earlier * Replace AST walk by interface * Make analysis sequence closer to master * Undo some * Undo debug * Show dependencies in log * null check * Update tests * Make sure factory looks in correct scope Enable assertions in tests * Separate stub dependencies * Don't try and merge from empty stub * Match generic parameters by name * Use qualified names in generics Minor tests cleanup * Back to names * Update tests * usings * Fix dependencies for stub * Null check * PR feedback * PR feedback + fix merge issue in generics * Test fixes + PR feedback * Baseline updates * PR feedback * Restore original order * Order * Merge issues * Make cancellation closer to original * Pass cancellation to module walker * Simplify * Upgrade to 3.0 * Undo change + .NET 3 * Get back to 2.0 * Update components * Display caching level Round time to reasonble fractions. * Fix null ref due to conflicting changes * Null ref fix * Simplify * Pass AST whish is required during analysis when module does not have its own yet. * tensorflow test * Fix from import model Add tensorflow test * Undo debug * Improve goto def in import and fix class base processing (#1529) * Update signing project to 3.0 * Upgrade to .NET 3.0 (#1521) * Upgrade to 3.0 * Get back to 2.0 * Add global.json for .NET SDK * Ensure FileInfo is not relative in DirectoryInfoProxy (#1519) * Ensure FileInfo is not relative in DirectoryInfoProxy * Add tests * Hadle import * Update dependencies (#1526) * Handle from import * Add test * Fix bad find and replace in AssertionsUtilities (#1527) * Merge issues * Add test (fails) * using * Fix merge of classes * Fix regression in bases Split merging into separate class * Fix null refs * Make common CanUpdateAnalysis check * Usings * Test fix * Remove CompletionItemKind.None * Add info to asserts * PR feedback * Undo test change, enable class reassignment Fix stub merge for TypeVar * Prevent merge of unrelated types * more tests * Fix variable update * Null check * Baselines * Move analysis complete event * More stable enumeration * Improve restoring imports in tests * Make extension for tests * Add URI null check * Fix completion typo * Add assert on unresolved modules * Add more details to assert * Using * To 3.7 * Implement check for big/little endian * Sync * Synchronization Fix import model restore * Add hard assert * Fix exception * Fix typo * Add lock --- .gitattributes | 1 + build/Common.Build.Core.settings | 4 +- .../Impl/Analyzer/Definitions/IAnalyzable.cs | 7 + .../Evaluation/ExpressionEval.Callables.cs | 1 - .../Evaluation/ExpressionEval.Collections.cs | 14 +- .../Evaluation/ExpressionEval.Constants.cs | 2 +- .../Evaluation/ExpressionEval.Generics.cs | 34 +- .../Evaluation/ExpressionEval.Operators.cs | 4 +- .../Analyzer/Evaluation/ExpressionEval.cs | 32 +- .../Analyzer/Handlers/AssignmentHandler.cs | 6 +- .../Analyzer/Handlers/ConditionalHandler.cs | 2 +- .../Analyzer/Handlers/FromImportHandler.cs | 22 +- .../Impl/Analyzer/Handlers/ImportHandler.cs | 2 +- .../Ast/Impl/Analyzer/LibraryAnalysis.cs | 3 - .../Ast/Impl/Analyzer/ModuleWalker.cs | 151 +------ .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 12 +- .../Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 84 +--- .../Impl/Analyzer/PythonAnalyzerSession.cs | 233 ++++++---- src/Analysis/Ast/Impl/Analyzer/StubMerger.cs | 308 +++++++++++++ .../Impl/Analyzer/Symbols/ClassEvaluator.cs | 19 +- .../Analyzer/Symbols/FunctionEvaluator.cs | 14 +- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 40 +- src/Analysis/Ast/Impl/Caching/CacheFolders.cs | 38 +- .../Definitions/ICacheFolderService.cs | 21 + .../Definitions/IModuleDatabaseService.cs | 64 +++ src/Analysis/Ast/Impl/Caching/StubCache.cs | 9 +- .../Ast/Impl/Definitions/AnalysisOptions.cs | 24 + .../Impl/Dependencies/DependencyCollector.cs | 77 ++++ .../Ast/Impl/Dependencies/DependencyWalker.cs | 55 +++ .../Impl/Dependencies/IDependencyProvider.cs | 29 ++ .../Impl/Dependencies/IDependencyResolver.cs | 1 - .../Ast/Impl/Documents/DocumentBuffer.cs | 53 +-- .../Impl/Documents/RunningDocumentTable.cs | 4 +- .../Ast/Impl/Extensions/AnalysisExtensions.cs | 4 +- .../Impl/Extensions/ArgumentSetExtensions.cs | 8 +- .../Impl/Extensions/IfStatementExtensions.cs | 19 + .../Ast/Impl/Extensions/NodeExtensions.cs | 5 +- .../Impl/Extensions/PythonClassExtensions.cs | 4 +- .../Extensions/PythonFunctionExtensions.cs | 21 +- .../Impl/Extensions/PythonModuleExtensions.cs | 2 +- .../Impl/Extensions/PythonTypeExtensions.cs | 5 +- .../Impl/Extensions/PythonWalkerExtensions.cs | 10 +- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 2 +- .../Modules/CompiledBuiltinPythonModule.cs | 4 +- .../Ast/Impl/Modules/CompiledPythonModule.cs | 6 +- .../Modules/Definitions/IModuleManagement.cs | 21 +- .../Definitions/ModuleCreationOptions.cs | 5 + .../Impl/Modules/Definitions/ModuleType.cs | 1 + .../Ast/Impl/Modules/DependencyProvider.cs | 52 +++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 67 ++- .../Ast/Impl/Modules/PythonVariableModule.cs | 27 +- .../Resolution/MainModuleResolution.cs | 59 ++- .../Resolution/ModuleResolutionBase.cs | 15 +- .../Modules/Resolution/TypeshedResolution.cs | 2 - .../Ast/Impl/Modules/SpecializedModule.cs | 7 +- .../Ast/Impl/Modules/StubPythonModule.cs | 2 +- .../Ast/Impl/Properties/AssemblyInfo.cs | 2 + .../BuiltinsSpecializations.cs | 32 +- .../Ast/Impl/Specializations/Specialized.cs | 6 +- ...ClassParameter.cs => IGenericClassBase.cs} | 2 +- .../Definitions/IGenericTypeParameter.cs | 3 + .../Definitions/ITypingNamedTupleType.cs | 1 - .../Specializations/Typing/Types/AnyType.cs | 1 + ...cClassParameter.cs => GenericClassBase.cs} | 28 +- .../Typing/Types/GenericType.cs | 66 ++- .../Typing/Types/GenericTypeParameter.cs | 67 ++- .../Typing/Types/NamedTupleType.cs | 33 +- .../Typing/Types/OptionalType.cs | 2 + .../Specializations/Typing/Types/TypeAlias.cs | 1 + .../Typing/Types/TypingDictionaryType.cs | 8 +- .../Typing/Types/TypingIteratorType.cs | 7 +- .../Typing/Types/TypingListType.cs | 8 +- .../Typing/Types/TypingTupleType.cs | 14 +- .../Specializations/Typing/TypingModule.cs | 69 +-- .../Typing/TypingTypeFactory.cs | 10 +- .../Typing/Values/TypingType.cs | 2 + src/Analysis/Ast/Impl/Types/ArgumentSet.cs | 14 +- .../Types/Collections/PythonCollectionType.cs | 71 +-- .../Types/Collections/PythonDictionaryType.cs | 4 +- .../Types/Collections/PythonIteratorType.cs | 6 +- .../Types/Definitions/IPythonClassMember.cs | 2 +- .../Types/Definitions/IPythonClassType.cs | 4 +- .../Definitions/IPythonCollectionType.cs | 2 +- .../Definitions/IPythonFunctionOverload.cs | 1 - .../Impl/Types/Definitions/IPythonModule.cs | 20 +- .../Types/Definitions/IPythonPropertyType.cs | 5 + .../Ast/Impl/Types/Definitions/IPythonType.cs | 5 + src/Analysis/Ast/Impl/Types/LocatedMember.cs | 3 + src/Analysis/Ast/Impl/Types/Location.cs | 5 +- src/Analysis/Ast/Impl/Types/ParameterInfo.cs | 2 +- .../Impl/Types/PythonClassType.Generics.cs | 95 ++-- .../Ast/Impl/Types/PythonClassType.cs | 71 ++- .../Ast/Impl/Types/PythonFunctionOverload.cs | 29 +- .../Ast/Impl/Types/PythonFunctionType.cs | 40 +- .../Ast/Impl/Types/PythonPropertyType.cs | 24 +- src/Analysis/Ast/Impl/Types/PythonType.cs | 21 +- .../Ast/Impl/Types/PythonTypeWrapper.cs | 24 +- .../Ast/Impl/Types/PythonUnionType.cs | 14 +- .../Ast/Impl/Utilities/ReentrancyGuard.cs | 4 +- .../Values/Collections/PythonCollection.cs | 2 +- .../Values/Collections/PythonDictionary.cs | 16 +- .../Collections/PythonInstanceIterator.cs | 2 +- .../Impl/Values/Collections/PythonIterator.cs | 2 +- .../Values/Collections/PythonTypeIterator.cs | 2 +- src/Analysis/Ast/Test/AnalysisTestBase.cs | 9 +- src/Analysis/Ast/Test/BasicTests.cs | 54 --- src/Analysis/Ast/Test/BuiltinsTests.cs | 99 ++++ src/Analysis/Ast/Test/ClassesTests.cs | 55 +++ src/Analysis/Ast/Test/ConditionalsTests.cs | 32 ++ src/Analysis/Ast/Test/DocumentBufferTests.cs | 2 +- src/Analysis/Ast/Test/EnumTests.cs | 48 ++ .../Test/FluentAssertions/MemberAssertions.cs | 93 +++- .../PythonFunctionAssertions.cs | 13 +- .../PythonFunctionOverloadAssertions.cs | 20 + src/Analysis/Ast/Test/FunctionTests.cs | 33 +- src/Analysis/Ast/Test/GenericsTests.cs | 30 ++ src/Analysis/Ast/Test/ImportTests.cs | 9 - src/Analysis/Ast/Test/LibraryTests.cs | 6 +- src/Analysis/Ast/Test/LintGenericTests.cs | 20 +- src/Analysis/Ast/Test/LintNoQATests.cs | 7 +- .../Microsoft.Python.Analysis.Tests.csproj | 3 + .../Ast/Test/PathClassificationTests.cs | 252 ----------- src/Analysis/Ast/Test/PepHintTests.cs | 2 +- .../Ast/Test/Properties/AssemblyInfo.cs | 18 + src/Analysis/Ast/Test/ScrapeTests.cs | 3 +- src/Analysis/Ast/Test/StubMergeTests.cs | 66 +++ src/Analysis/Ast/Test/TypeshedTests.cs | 2 +- src/Analysis/Ast/Test/TypingTests.cs | 32 +- .../Impl/DependencyResolution/AstUtilities.cs | 11 +- .../Impl/DependencyResolution/ImportRoot.cs | 3 +- .../Impl/DependencyResolution/ModuleImport.cs | 1 + .../Interpreter/InterpreterConfiguration.cs | 88 +--- .../Core/Impl/Interpreter/LibraryType.cs | 38 ++ .../Core/Impl/Interpreter/ModulePath.cs | 4 - .../Impl/Interpreter/PythonLibraryPath.cs | 57 +-- .../Core/Impl/Properties/AssemblyInfo.cs | 1 + .../DependencyCollectorExtensions.cs | 41 ++ .../Impl/Extensions/IndexSpanExtensions.cs | 29 ++ .../Microsoft.Python.Analysis.Caching.csproj | 44 ++ src/Caching/Impl/Models/CallableModel.cs | 86 ++++ src/Caching/Impl/Models/ClassModel.cs | 201 +++++++++ .../Impl/Models/DottedNameModel.cs} | 15 +- src/Caching/Impl/Models/FromImportModel.cs | 26 ++ .../Impl/Models/FunctionAttributes.cs} | 17 +- src/Caching/Impl/Models/FunctionModel.cs | 75 ++++ .../Impl/Models/GenericParameterValueModel.cs | 32 ++ src/Caching/Impl/Models/ImportModel.cs | 24 + src/Caching/Impl/Models/IndexSpanModel.cs | 28 ++ src/Caching/Impl/Models/MemberModel.cs | 61 +++ src/Caching/Impl/Models/ModuleModel.cs | 218 +++++++++ src/Caching/Impl/Models/NamedTupleModel.cs | 57 +++ src/Caching/Impl/Models/NewLineModel.cs | 24 + src/Caching/Impl/Models/OverloadModel.cs | 21 + src/Caching/Impl/Models/ParameterModel.cs | 25 ++ src/Caching/Impl/Models/PropertyModel.cs | 47 ++ src/Caching/Impl/Models/TypeVarModel.cs | 55 +++ src/Caching/Impl/Models/VariableModel.cs | 59 +++ src/Caching/Impl/ModuleDatabase.cs | 256 +++++++++++ src/Caching/Impl/ModuleFactory.cs | 240 ++++++++++ src/Caching/Impl/ModuleUniqueId.cs | 119 +++++ src/Caching/Impl/Properties/AssemblyInfo.cs | 19 + src/Caching/Impl/PythonDbModule.cs | 52 +++ src/Caching/Impl/QualifiedNameParts.cs | 37 ++ src/Caching/Impl/RestoredGlobalScope.cs | 88 ++++ src/Caching/Impl/TypeNames.cs | 163 +++++++ src/Caching/Test/AnalysisCachingTestBase.cs | 94 ++++ src/Caching/Test/AssemblySetup.cs | 34 ++ src/Caching/Test/ClassesTests.cs | 154 +++++++ src/Caching/Test/CoreTests.cs | 114 +++++ .../Test/Files/ClassOwnDocumentation.json | 202 +++++++++ src/Caching/Test/Files/MemberLocations.json | 373 +++++++++++++++ src/Caching/Test/Files/NestedClasses.json | 424 ++++++++++++++++++ src/Caching/Test/Files/SmokeTest.json | 321 +++++++++++++ src/Caching/Test/Files/VersionHandling2.json | 148 ++++++ src/Caching/Test/Files/VersionHandling3.json | 160 +++++++ .../FluentAssertions/AssertionsFactory.cs | 37 ++ src/Caching/Test/LibraryModulesTests.cs | 318 +++++++++++++ ...osoft.Python.Analysis.Caching.Tests.csproj | 42 ++ src/Caching/Test/ReferencesTests.cs | 119 +++++ src/Core/Impl/Extensions/StringExtensions.cs | 11 + src/Core/Impl/IO/DirectoryInfoProxy.cs | 2 +- src/Core/Impl/OS/IOSPlatform.cs | 1 + src/Core/Impl/OS/OSPlatform.cs | 2 + src/Core/Impl/OS/ProcessServices.cs | 22 +- src/Core/Impl/Text/ILocationConverter.cs | 26 ++ src/Core/Test/PriorityProducerConsumerTest.cs | 234 +++++----- .../Impl/Completion/CompletionItemSource.cs | 4 +- .../Impl/Completion/ExpressionCompletion.cs | 6 +- .../Impl/Documentation/DocstringConverter.cs | 2 +- .../Impl/Implementation/Server.cs | 5 +- .../Impl/LanguageServer.Configuration.cs | 16 +- .../Microsoft.Python.LanguageServer.csproj | 1 + src/LanguageServer/Impl/Protocol/Enums.cs | 4 +- src/LanguageServer/Impl/Resources.Designer.cs | 18 + src/LanguageServer/Impl/Resources.resx | 6 + .../Impl/Sources/DefinitionSource.cs | 144 ++++-- .../Impl/Sources/ReferenceSource.cs | 4 +- src/LanguageServer/Test/CompletionTests.cs | 2 +- .../Test/GoToDefinitionTests.cs | 110 ++++- src/LanguageServer/Test/HoverTests.cs | 2 +- src/LanguageServer/Test/IndexManagerTests.cs | 245 +++++----- src/LanguageServer/Test/IndexParserTests.cs | 46 +- src/LanguageServer/Test/SymbolIndexTests.cs | 174 +++---- src/PLS.sln | 14 + src/Parsing/Impl/Ast/PythonAst.cs | 4 +- .../Impl/Ast/SourceLocationExtensions.cs | 14 +- src/Parsing/Impl/Resources.Designer.cs | 2 +- src/Parsing/Impl/Tokens/NewLineKind.cs | 24 + .../Impl/Tokens/NewLineKindExtensions.cs | 39 ++ src/Parsing/Impl/Tokens/NewLineLocation.cs | 120 +++++ src/Parsing/Impl/{ => Tokens}/Token.cs | 0 .../Impl/{ => Tokens}/TokenCategory.cs | 0 src/Parsing/Impl/{ => Tokens}/TokenInfo.cs | 0 .../Impl/{ => Tokens}/TokenKind.Generated.cs | 0 .../Impl/{ => Tokens}/TokenTriggers.cs | 0 src/Parsing/Impl/{ => Tokens}/Tokenizer.cs | 130 ------ .../Impl/{ => Tokens}/TokenizerOptions.cs | 0 src/Parsing/Test/PythonInstallPathResolver.cs | 1 - src/Parsing/Test/PythonVersion.cs | 2 - src/Parsing/Test/PythonVersions.cs | 51 +-- .../Test/UnixPythonInstallPathResolver.cs | 4 - .../Test/WindowsPythonInstallPathResolver.cs | 34 -- src/UnitTests/Core/Impl/Baseline.cs | 125 ++++++ 223 files changed, 8043 insertions(+), 1988 deletions(-) create mode 100644 .gitattributes create mode 100644 src/Analysis/Ast/Impl/Analyzer/StubMerger.cs create mode 100644 src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs create mode 100644 src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs create mode 100644 src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs create mode 100644 src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs create mode 100644 src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs create mode 100644 src/Analysis/Ast/Impl/Modules/DependencyProvider.cs rename src/Analysis/Ast/Impl/Specializations/Typing/Definitions/{IGenericClassParameter.cs => IGenericClassBase.cs} (95%) rename src/Analysis/Ast/Impl/Specializations/Typing/Types/{GenericClassParameter.cs => GenericClassBase.cs} (55%) create mode 100644 src/Analysis/Ast/Test/BuiltinsTests.cs create mode 100644 src/Analysis/Ast/Test/EnumTests.cs create mode 100644 src/Analysis/Ast/Test/Properties/AssemblyInfo.cs create mode 100644 src/Analysis/Ast/Test/StubMergeTests.cs create mode 100644 src/Analysis/Core/Impl/Interpreter/LibraryType.cs create mode 100644 src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs create mode 100644 src/Caching/Impl/Extensions/IndexSpanExtensions.cs create mode 100644 src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj create mode 100644 src/Caching/Impl/Models/CallableModel.cs create mode 100644 src/Caching/Impl/Models/ClassModel.cs rename src/{Analysis/Ast/Impl/Types/Definitions/IPythonFile.cs => Caching/Impl/Models/DottedNameModel.cs} (71%) create mode 100644 src/Caching/Impl/Models/FromImportModel.cs rename src/{Analysis/Ast/Impl/Extensions/QualifiedNameExtensions.cs => Caching/Impl/Models/FunctionAttributes.cs} (60%) create mode 100644 src/Caching/Impl/Models/FunctionModel.cs create mode 100644 src/Caching/Impl/Models/GenericParameterValueModel.cs create mode 100644 src/Caching/Impl/Models/ImportModel.cs create mode 100644 src/Caching/Impl/Models/IndexSpanModel.cs create mode 100644 src/Caching/Impl/Models/MemberModel.cs create mode 100644 src/Caching/Impl/Models/ModuleModel.cs create mode 100644 src/Caching/Impl/Models/NamedTupleModel.cs create mode 100644 src/Caching/Impl/Models/NewLineModel.cs create mode 100644 src/Caching/Impl/Models/OverloadModel.cs create mode 100644 src/Caching/Impl/Models/ParameterModel.cs create mode 100644 src/Caching/Impl/Models/PropertyModel.cs create mode 100644 src/Caching/Impl/Models/TypeVarModel.cs create mode 100644 src/Caching/Impl/Models/VariableModel.cs create mode 100644 src/Caching/Impl/ModuleDatabase.cs create mode 100644 src/Caching/Impl/ModuleFactory.cs create mode 100644 src/Caching/Impl/ModuleUniqueId.cs create mode 100644 src/Caching/Impl/Properties/AssemblyInfo.cs create mode 100644 src/Caching/Impl/PythonDbModule.cs create mode 100644 src/Caching/Impl/QualifiedNameParts.cs create mode 100644 src/Caching/Impl/RestoredGlobalScope.cs create mode 100644 src/Caching/Impl/TypeNames.cs create mode 100644 src/Caching/Test/AnalysisCachingTestBase.cs create mode 100644 src/Caching/Test/AssemblySetup.cs create mode 100644 src/Caching/Test/ClassesTests.cs create mode 100644 src/Caching/Test/CoreTests.cs create mode 100644 src/Caching/Test/Files/ClassOwnDocumentation.json create mode 100644 src/Caching/Test/Files/MemberLocations.json create mode 100644 src/Caching/Test/Files/NestedClasses.json create mode 100644 src/Caching/Test/Files/SmokeTest.json create mode 100644 src/Caching/Test/Files/VersionHandling2.json create mode 100644 src/Caching/Test/Files/VersionHandling3.json create mode 100644 src/Caching/Test/FluentAssertions/AssertionsFactory.cs create mode 100644 src/Caching/Test/LibraryModulesTests.cs create mode 100644 src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj create mode 100644 src/Caching/Test/ReferencesTests.cs create mode 100644 src/Core/Impl/Text/ILocationConverter.cs create mode 100644 src/Parsing/Impl/Tokens/NewLineKind.cs create mode 100644 src/Parsing/Impl/Tokens/NewLineKindExtensions.cs create mode 100644 src/Parsing/Impl/Tokens/NewLineLocation.cs rename src/Parsing/Impl/{ => Tokens}/Token.cs (100%) rename src/Parsing/Impl/{ => Tokens}/TokenCategory.cs (100%) rename src/Parsing/Impl/{ => Tokens}/TokenInfo.cs (100%) rename src/Parsing/Impl/{ => Tokens}/TokenKind.Generated.cs (100%) rename src/Parsing/Impl/{ => Tokens}/TokenTriggers.cs (100%) rename src/Parsing/Impl/{ => Tokens}/Tokenizer.cs (95%) rename src/Parsing/Impl/{ => Tokens}/TokenizerOptions.cs (100%) create mode 100644 src/UnitTests/Core/Impl/Baseline.cs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..587b786e4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.json -diff diff --git a/build/Common.Build.Core.settings b/build/Common.Build.Core.settings index a38d4dbea..cc11ec495 100644 --- a/build/Common.Build.Core.settings +++ b/build/Common.Build.Core.settings @@ -9,8 +9,8 @@ + Please do not put Windows-specific or desktop-specific settings + here (these include C#, C++ or VS SDK settings and targets --> Debug 10.0 diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 8d4b9751d..2b9182661 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -13,11 +13,18 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Dependencies; + namespace Microsoft.Python.Analysis.Analyzer { /// /// Represents document that can be analyzed asynchronously. /// internal interface IAnalyzable { + /// + /// Returns object that can calculate dependencies of this entry. + /// + IDependencyProvider DependencyProvider { get; } + /// /// Notifies document that analysis is about to begin. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index b10ab0e68..00baa9b4d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -17,7 +17,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs index 077017c9c..e641849c9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs @@ -62,7 +62,7 @@ public IMember GetValueFromList(ListExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateList(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize); + return PythonCollectionType.CreateList(Module, contents, exact: expression.Items.Count <= MaxCollectionSize); } public IMember GetValueFromDictionary(DictionaryExpression expression) { @@ -72,7 +72,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) { var value = GetValueFromExpression(item.SliceStop) ?? UnknownType; contents[key] = value; } - return new PythonDictionary(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize); + return new PythonDictionary(Module, contents, exact: expression.Items.Count <= MaxCollectionSize); } private IMember GetValueFromTuple(TupleExpression expression) { @@ -81,7 +81,7 @@ private IMember GetValueFromTuple(TupleExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateTuple(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize); + return PythonCollectionType.CreateTuple(Module, contents, exact: expression.Items.Count <= MaxCollectionSize); } public IMember GetValueFromSet(SetExpression expression) { @@ -90,7 +90,7 @@ public IMember GetValueFromSet(SetExpression expression) { var value = GetValueFromExpression(item) ?? UnknownType; contents.Add(value); } - return PythonCollectionType.CreateSet(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize); + return PythonCollectionType.CreateSet(Module, contents, exact: expression.Items.Count <= MaxCollectionSize); } public IMember GetValueFromGenerator(GeneratorExpression expression) { @@ -110,14 +110,14 @@ public IMember GetValueFromComprehension(Comprehension node) { switch (node) { case ListComprehension lc: var v1 = GetValueFromExpression(lc.Item) ?? UnknownType; - return PythonCollectionType.CreateList(Interpreter, new[] { v1 }); + return PythonCollectionType.CreateList(Module, new[] { v1 }); case SetComprehension sc: var v2 = GetValueFromExpression(sc.Item) ?? UnknownType; - return PythonCollectionType.CreateSet(Interpreter, new[] { v2 }); + return PythonCollectionType.CreateSet(Module, new[] { v2 }); case DictionaryComprehension dc: var k = GetValueFromExpression(dc.Key) ?? UnknownType; var v = GetValueFromExpression(dc.Value) ?? UnknownType; - return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary { { k, v } }); + return new PythonDictionary(new PythonDictionaryType(Interpreter.ModuleResolution.BuiltinsModule), new Dictionary { { k, v } }); } return UnknownType; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs index 53376c5a0..52185a35c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { internal sealed partial class ExpressionEval { - public IPythonInstance GetConstantFromLiteral(Expression expr, LookupOptions options) { + public IPythonInstance GetConstantFromLiteral(Expression expr) { if (expr is ConstantExpression ce) { switch (ce.Value) { case string s: diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs index ba77072ef..8ff168e52 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs @@ -62,11 +62,12 @@ private IMember GetValueFromGeneric(IMember target, Expression expr) { } /// - /// Returns whether the arguments to Generic are valid + /// Determines if arguments to Generic are valid /// + // TODO: move check to GenericClassBase. This requires extensive changes to SpecificTypeConstructor. private bool GenericClassParameterValid(IReadOnlyList genericTypeArgs, IReadOnlyList args, Expression expr) { - // All arguments to Generic must be type parameters - // e.g. Generic[T, str] throws a runtime error + // All arguments to Generic must be type parameters + // e.g. Generic[T, str] throws a runtime error if (genericTypeArgs.Count != args.Count) { ReportDiagnostics(Module.Uri, new DiagnosticsEntry( Resources.GenericNotAllTypeParameters, @@ -77,14 +78,14 @@ private bool GenericClassParameterValid(IReadOnlyList gen return false; } - // All arguments to Generic must be distinct + // All arguments to Generic must be distinct if (genericTypeArgs.Distinct().Count() != genericTypeArgs.Count) { ReportDiagnostics(Module.Uri, new DiagnosticsEntry( - Resources.GenericNotAllUnique, - GetLocation(expr).Span, - ErrorCodes.TypingGenericArguments, - Severity.Warning, - DiagnosticSource.Analysis)); + Resources.GenericNotAllUnique, + GetLocation(expr).Span, + ErrorCodes.TypingGenericArguments, + Severity.Warning, + DiagnosticSource.Analysis)); return false; } @@ -97,24 +98,19 @@ private bool GenericClassParameterValid(IReadOnlyList gen /// (if the former) on specific type (if the latter). /// private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList args, Expression expr) { + // TODO: move check to GenericClassBase. This requires extensive changes to SpecificTypeConstructor. if (gt.Name.EqualsOrdinal("Generic")) { var genericTypeArgs = args.OfType().ToArray(); if (!GenericClassParameterValid(genericTypeArgs, args, expr)) { return UnknownType; } - - // Generic[T1, T2, ...] expression. Create generic base for the class. - return new GenericClassParameter(genericTypeArgs, Module); - } - - // For other types just use supplied arguments - if (args.Count > 0) { - return gt.CreateSpecificType(new ArgumentSet(args, expr, this)); + // Generic[T1, T2, ...] expression. Create generic base for the class. + return new GenericClassBase(genericTypeArgs, Module.Interpreter); } - return UnknownType; + // For other types just use supplied arguments + return args.Count > 0 ? gt.CreateSpecificType(new ArgumentSet(args, expr, this)) : UnknownType; } - private IReadOnlyList EvaluateIndex(IndexExpression expr) { var indices = new List(); if (expr.Index is TupleExpression tex) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs index e5dfe0330..a21af69f9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs @@ -105,9 +105,9 @@ private IMember GetValueFromBinaryOp(Expression expr) { switch (leftTypeId) { case BuiltinTypeId.List: - return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc); + return PythonCollectionType.CreateConcatenatedList(Module, lc, rc); case BuiltinTypeId.Tuple: - return PythonCollectionType.CreateConcatenatedTuple(Module.Interpreter, lc, rc); + return PythonCollectionType.CreateConcatenatedTuple(Module, lc, rc); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index c05a0a76a..dceef6fd7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -55,6 +55,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable(); public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } + public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; @@ -188,7 +189,7 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options = L m = null; break; default: - m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options); + m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr); break; } if (m == null) { @@ -236,22 +237,12 @@ private IMember GetValueFromMember(MemberExpression expr) { return null; } - IPythonInstance instance = null; var m = GetValueFromExpression(expr.Target); - if (m is IPythonType typeInfo) { - var member = typeInfo.GetMember(expr.Name); - // If container is class/type info rather than the instance, then the method is an unbound function. - // Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C. - if (member is PythonFunctionType f && !f.IsStatic && !f.IsClassMethod) { - f.AddReference(GetLocationOfName(expr)); - return f.ToUnbound(); - } - instance = new PythonInstance(typeInfo); + if (m == null) { + return UnknownType; } - instance = instance ?? m as IPythonInstance; - var type = m?.GetPythonType(); // Try inner type - + var type = m.GetPythonType(); var value = type?.GetMember(expr.Name); type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr)); @@ -259,6 +250,19 @@ private IMember GetValueFromMember(MemberExpression expr) { return value; } + IPythonInstance instance = null; + if (m == type) { + // If container is class/type info rather than the instance, then the method is an unbound function. + // Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C. + if (value is PythonFunctionType f && f.DeclaringType != null && !f.IsStatic && !f.IsClassMethod) { + f.AddReference(GetLocationOfName(expr)); + return f.ToUnbound(); + } + instance = new PythonInstance(type); + } + + instance = instance ?? m as IPythonInstance; + // Class type GetMember returns a type. However, class members are // mostly instances (consider self.x = 1, x is an instance of int). // However, it is indeed possible to have them as types, like in diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 875313eeb..a1582af45 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -58,14 +58,14 @@ public void HandleAssignment(AssignmentStatement node) { } foreach (var ne in node.Left.OfType()) { + IScope scope; if (Eval.CurrentScope.NonLocals[ne.Name] != null) { - Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); + Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal); scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } - if (Eval.CurrentScope.Globals[ne.Name] != null) { - Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); + Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global); scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs index 93c82cfd6..108bb0b31 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs @@ -25,6 +25,6 @@ public ConditionalHandler(AnalysisWalker walker) : base(walker) { } public bool HandleIf(IfStatement node) - => node.WalkIfWithSystemConditions(Walker, Ast.LanguageVersion, _platformService.IsWindows); + => node.WalkIfWithSystemConditions(Walker, Ast.LanguageVersion, _platformService); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 50edd0261..3374d04b2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -68,11 +68,13 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor if (!string.IsNullOrEmpty(memberName)) { var nameExpression = asNames[i] ?? names[i]; var variableName = nameExpression?.Name ?? memberName; - var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - var exported = variable ?? variableModule.GetMember(memberName); - var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); - // Do not allow imported variables to override local declarations - Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex)); + if (!string.IsNullOrEmpty(variableName)) { + var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; + var exported = variable ?? variableModule.GetMember(memberName); + var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + // Do not allow imported variables to override local declarations + Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex)); + } } } } @@ -84,7 +86,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is } // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. - var memberNames = isImplicitPackage + var memberNames = isImplicitPackage ? variableModule.GetMemberNames() : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); @@ -109,7 +111,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is private bool CanOverwriteVariable(string name, int importPosition) { var v = Eval.CurrentScope.Variables[name]; - if(v == null) { + if (v == null) { return true; // Variable does not exist } // Allow overwrite if import is below the variable. Consider @@ -118,10 +120,10 @@ private bool CanOverwriteVariable(string name, int importPosition) { // from A import * # brings another x // x = 3 var references = v.References.Where(r => r.DocumentUri == Module.Uri).ToArray(); - if(references.Length == 0) { + if (references.Length == 0) { // No references to the variable in this file - the variable // is imported from another module. OK to overwrite. - return true; + return true; } var firstAssignmentPosition = references.Min(r => r.Span.ToIndexSpan(Ast).Start); return firstAssignmentPosition < importPosition; @@ -151,7 +153,7 @@ private void SpecializeFuture(FromImportStatement node) { var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function"); if (printNameExpression != null) { var fn = new PythonFunctionType("print", new Location(Module), null, string.Empty); - var o = new PythonFunctionOverload(fn.Name, new Location(Module)); + var o = new PythonFunctionOverload(fn, new Location(Module)); var parameters = new List { new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null), new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null), diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index af71f6556..b18b14883 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -180,7 +180,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py return variableModule; } - variableModule = new PythonVariableModule(fullName, Interpreter); + variableModule = new PythonVariableModule(fullName, Eval.Interpreter); _variableModules[fullName] = variableModule; parentModule?.AddChildModule(memberName, variableModule); return variableModule; diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index bf8f7df93..a68fe2642 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -15,12 +15,9 @@ using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing.Ast; diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 778bfb055..68f7a1cb1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -16,10 +16,9 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; @@ -32,30 +31,36 @@ namespace Microsoft.Python.Analysis.Analyzer { internal class ModuleWalker : AnalysisWalker { private const string AllVariableName = "__all__"; private readonly IDocumentAnalysis _stubAnalysis; + private readonly CancellationToken _cancellationToken; // A hack to use __all__ export in the most simple case. private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) + public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken) : base(new ExpressionEval(services, module, ast)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; + _cancellationToken = CancellationToken.None; } public override bool Walk(NameExpression node) { if (Eval.CurrentScope == Eval.GlobalScope && node.Name == AllVariableName) { _allReferencesCount++; } + + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } public override bool Walk(AugmentedAssignStatement node) { HandleAugmentedAllAssign(node); + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } public override bool Walk(CallExpression node) { HandleAllAppendExtend(node); + _cancellationToken.ThrowIfCancellationRequested(); return base.Walk(node); } @@ -107,7 +112,7 @@ private void HandleAllAppendExtend(CallExpression node) { IPythonCollection values = null; switch (me.Name) { case "append": - values = PythonCollectionType.CreateList(Module.Interpreter, new List { v }, exact: true); + values = PythonCollectionType.CreateList(Module, new List { v }, exact: true); break; case "extend": values = v as IPythonCollection; @@ -129,7 +134,7 @@ private void ExtendAll(Node location, IPythonCollection values) { } var all = scope.Variables[AllVariableName]?.Value as IPythonCollection; - var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, all, values); + var list = PythonCollectionType.CreateConcatenatedList(Module, all, values); var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; Eval.DeclareVariable(AllVariableName, list, source, location); @@ -146,6 +151,7 @@ private bool IsHandleableAll(Node node) { public override bool Walk(PythonAst node) { Check.InvalidOperation(() => Ast == node, "walking wrong AST"); + _cancellationToken.ThrowIfCancellationRequested(); // Collect basic information about classes and functions in order // to correctly process forward references. Does not determine @@ -181,19 +187,23 @@ public override bool Walk(PythonAst node) { // Classes and functions are walked by their respective evaluators public override bool Walk(ClassDefinition node) { + _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.Evaluate(node); return false; } public override bool Walk(FunctionDefinition node) { + _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.Evaluate(node); return false; } public void Complete() { + _cancellationToken.ThrowIfCancellationRequested(); + SymbolTable.EvaluateAll(); SymbolTable.ReplacedByStubs.Clear(); - MergeStub(); + new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken); if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable) && variable?.Value is IPythonCollection collection && collection.IsExact) { @@ -209,134 +219,5 @@ public void Complete() { public GlobalScope GlobalScope => Eval.GlobalScope; public IReadOnlyList StarImportMemberNames { get; private set; } - - /// - /// Merges data from stub with the data from the module. - /// - /// - /// Functions are taken from the stub by the function walker since - /// information on the return type is needed during the analysis walk. - /// However, if the module is compiled (scraped), it often lacks some - /// of the definitions. Stub may contains those so we need to merge it in. - /// - private void MergeStub() { - if (Module.ModuleType == ModuleType.User) { - return; - } - // No stub, no merge. - if (_stubAnalysis == null) { - return; - } - - var builtins = Module.Interpreter.ModuleResolution.BuiltinsModule; - - // Note that scrape can pick up more functions than the stub contains - // Or the stub can have definitions that scraping had missed. Therefore - // merge is the combination of the two with the documentation coming - // from the library source of from the scraped module. - foreach (var v in _stubAnalysis.GlobalScope.Variables) { - var stubType = v.Value.GetPythonType(); - if (stubType.IsUnknown()) { - continue; - } - - var sourceVar = Eval.GlobalScope.Variables[v.Name]; - var sourceType = sourceVar?.Value.GetPythonType(); - - // If stub says 'Any' but we have better type, keep the current type. - if (!IsStubBetterType(sourceType, stubType)) { - continue; - } - - // If type does not exist in module, but exists in stub, declare it unless it is an import. - // If types are the classes, merge members. Otherwise, replace type from one from the stub. - switch (sourceType) { - case null: - // Nothing in sources, but there is type in the stub. Declare it. - if (v.Source == VariableSource.Declaration) { - Eval.DeclareVariable(v.Name, v.Value, v.Source); - } - break; - - case PythonClassType cls when Module.Equals(cls.DeclaringModule): - // If class exists and belongs to this module, add or replace - // its members with ones from the stub, preserving documentation. - // Don't augment types that do not come from this module. - // Do not replace __class__ since it has to match class type and we are not - // replacing class type, we are only merging members. - foreach (var name in stubType.GetMemberNames().Except(new[] { "__class__", "__base__", "__bases__", "__mro__", "mro" })) { - var stubMember = stubType.GetMember(name); - var member = cls.GetMember(name); - - var memberType = member?.GetPythonType(); - var stubMemberType = stubMember.GetPythonType(); - - if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType?.DeclaringModule)) { - continue; // Leave builtins alone. - } - - if (memberType?.DeclaringModule is SpecializedModule || stubMemberType?.DeclaringModule is SpecializedModule) { - continue; // Leave specialized modules like typing alone. - } - - if (!IsStubBetterType(memberType, stubMemberType)) { - continue; - } - - // Get documentation from the current type, if any, since stubs - // typically do not contain documentation while scraped code does. - TransferDocumentationAndLocation(memberType, stubMemberType); - cls.AddMember(name, stubMember, overwrite: true); - } - break; - - case IPythonModule _: - // We do not re-declare modules. - break; - - default: - // Re-declare variable with the data from the stub unless member is a module. - // Modules members that are modules should remain as they are, i.e. os.path - // should remain library with its own stub attached. - var stubModule = stubType.DeclaringModule; - if (!(stubType is IPythonModule) && !builtins.Equals(stubModule)) { - TransferDocumentationAndLocation(sourceType, stubType); - // TODO: choose best type between the scrape and the stub. Stub probably should always win. - var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source; - Eval.DeclareVariable(v.Name, v.Value, source); - } - break; - } - } - } - - private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubType) { - if (stubType.IsUnknown()) { - // Do not use worse types than what is in the module. - return false; - } - if (sourceType.IsUnknown()) { - return true; // Anything is better than unknowns. - } - // If stub says 'Any' but we have better type, keep the current type. - return !(stubType.DeclaringModule is TypingModule) || stubType.Name != "Any"; - } - - private static void TransferDocumentationAndLocation(IPythonType s, IPythonType d) { - if (s.IsUnknown()) { - return; // Do not transfer location of unknowns - } - // Documentation and location are always get transferred from module type - // to the stub type and never the other way around. This makes sure that - // we show documentation from the original module and goto definition - // navigates to the module source and not to the stub. - if (s != d && s is PythonType src && d is PythonType dst) { - var documentation = src.Documentation; - if (!string.IsNullOrEmpty(documentation)) { - dst.SetDocumentation(documentation); - } - dst.Location = src.Location; - } - } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 8037a5868..3a1994ee6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -59,7 +59,9 @@ public PythonAnalyzer(IServiceManager services, string cacheFolderPath = null) { _startNextSession = StartNextSession; _progress = new ProgressReporter(services.GetService()); - _services.AddService(new StubCache(_services, cacheFolderPath)); + + _services.AddService(new CacheFolderService(_services, cacheFolderPath)); + _services.AddService(new StubCache(_services)); } public void Dispose() { @@ -230,13 +232,15 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; - internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) - => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); + internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { + _analysisCompleteEvent.Set(); + AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); + } private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { _analysisCompleteEvent.Reset(); ActivityTracker.StartTracking(); - _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); + _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}"); var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index d1258f97b..02503c64c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -19,15 +19,13 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { - [DebuggerDisplay("{_module.Name}({_module.ModuleType})")] + [DebuggerDisplay("{_module.Name} : {_module.ModuleType}")] internal sealed class PythonAnalyzerEntry { private readonly object _syncObj = new object(); private TaskCompletionSource _analysisTcs; @@ -111,7 +109,7 @@ public bool CanUpdateAnalysis(int version, out IPythonModule module, out PythonA return true; } - return _analysisVersion <= version && !(_previousAnalysis is LibraryAnalysis); + return _analysisVersion <= version; } } @@ -248,20 +246,21 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i } private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { + var dependencies = new HashSet(); if (_bufferVersion > bufferVersion) { - return new HashSet(); + return dependencies; + } + + var dependencyProvider = (module as IAnalyzable)?.DependencyProvider; + var moduleDeps = dependencyProvider?.GetDependencies(ast); + if (moduleDeps != null) { + dependencies.UnionWith(moduleDeps); } - var walker = new DependencyWalker(module); - ast.Walk(walker); - var dependencies = walker.Dependencies; dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; } - private static bool Ignore(IModuleManagement moduleResolution, string name) - => moduleResolution.BuiltinModuleName.EqualsOrdinal(name) || moduleResolution.GetSpecializedModule(name) != null; - private void UpdateAnalysisTcs(int analysisVersion) { _analysisVersion = analysisVersion; if (_analysisTcs.Task.Status == TaskStatus.RanToCompletion) { @@ -273,69 +272,6 @@ private void UpdateAnalysisTcs(int analysisVersion) { } } - private class DependencyWalker : PythonWalker { - private readonly IPythonModule _module; - private readonly bool _isTypeshed; - private readonly IModuleManagement _moduleResolution; - private readonly PathResolverSnapshot _pathResolver; - - public HashSet Dependencies { get; } - - public DependencyWalker(IPythonModule module) { - _module = module; - _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed; - _moduleResolution = module.Interpreter.ModuleResolution; - _pathResolver = _isTypeshed - ? module.Interpreter.TypeshedResolution.CurrentPathResolver - : _moduleResolution.CurrentPathResolver; - - Dependencies = new HashSet(); - - if (module.Stub != null) { - Dependencies.Add(new AnalysisModuleKey(module.Stub)); - } - } - - public override bool Walk(ImportStatement import) { - var forceAbsolute = import.ForceAbsolute; - foreach (var moduleName in import.Names) { - var importNames = ImmutableArray.Empty; - foreach (var nameExpression in moduleName.Names) { - importNames = importNames.Add(nameExpression.Name); - var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); - HandleSearchResults(imports); - } - } - return false; - } - - public override bool Walk(FromImportStatement fromImport) { - var imports = _pathResolver.FindImports(_module.FilePath, fromImport); - HandleSearchResults(imports); - if (imports is IImportChildrenSource childrenSource) { - foreach (var name in fromImport.Names) { - if (childrenSource.TryGetChildImport(name.Name, out var childImport)) { - HandleSearchResults(childImport); - } - } - } - - return false; - } - - private void HandleSearchResults(IImportSearchResult searchResult) { - switch (searchResult) { - case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName): - Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed)); - return; - case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName): - Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed)); - return; - default: - return; - } - } - } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 787327622..af049c36a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -21,6 +21,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -48,6 +49,7 @@ internal sealed class PythonAnalyzerSession { private readonly IPythonAnalyzer _analyzer; private readonly ILogger _log; private readonly bool _forceGC; + private readonly IModuleDatabaseService _moduleDatabaseService; private State _state; private bool _isCanceled; @@ -88,6 +90,7 @@ public PythonAnalyzerSession(IServiceManager services, _diagnosticsService = _services.GetService(); _analyzer = _services.GetService(); _log = _services.GetService(); + _moduleDatabaseService = _services.GetService(); _progress = progress; } @@ -149,6 +152,7 @@ private async Task StartAsync() { _progress.ReportRemaining(remaining); if (isFinal) { var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + totalMilliseconds = Math.Round(totalMilliseconds, 2); (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); } @@ -174,6 +178,7 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai return; } + elapsed = Math.Round(elapsed, 2); if (remaining == 0) { logger.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms."); } else if (remaining < originalRemaining) { @@ -202,7 +207,13 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath); - if (Interlocked.Increment(ref _runningTasks) >= _maxTaskRunning || _walker.Remaining == 1) { + var taskLimitReached = false; + lock (_syncObj) { + _runningTasks++; + taskLimitReached = _runningTasks >= _maxTaskRunning || _walker.Remaining == 1; + } + + if (taskLimitReached) { RunAnalysis(node, stopWatch); } else { ace.AddOne(); @@ -213,17 +224,11 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { await ace.WaitAsync(_analyzerCancellationToken); lock (_syncObj) { - isCanceled = _isCanceled; - } - - if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { - Interlocked.Exchange(ref _runningTasks, 0); - - if (!isCanceled) { - _analysisCompleteEvent.Set(); + if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { + Debug.Assert(_runningTasks == 0); + } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { + _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } - } else if (!isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { - _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } return remaining; @@ -233,7 +238,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node, IDocumentAnalysis currentAnalysis) => !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion; - private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) + private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null); private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) @@ -247,23 +252,9 @@ private Task StartAnalysis(IDependencyChainNode node, Async private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { try { var entry = node.Value; - - if (!entry.CanUpdateAnalysis(_walker.Version, out var module, out var ast, out var currentAnalysis)) { - if (IsAnalyzedLibraryInLoop(node, currentAnalysis)) { - return; - } else if (ast == default) { - if (currentAnalysis == default) { - // Entry doesn't have ast yet. There should be at least one more session. - Cancel(); - } else { - Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); - } - } - - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + if (!CanUpdateAnalysis(entry, node, _walker.Version, out var module, out var ast, out var currentAnalysis)) { return; } - var startTime = stopWatch.Elapsed; AnalyzeEntry(node, entry, module, ast, _walker.Version); @@ -278,41 +269,23 @@ private void Analyze(IDependencyChainNode node, AsyncCountd } finally { node.MoveNext(); - bool isCanceled; lock (_syncObj) { - isCanceled = _isCanceled; - } - - if (!isCanceled) { - _progress.ReportRemaining(_walker.Remaining); + if (!_isCanceled) { + _progress.ReportRemaining(_walker.Remaining); + } + _runningTasks--; + ace?.Signal(); } - - Interlocked.Decrement(ref _runningTasks); - ace?.Signal(); } } private void AnalyzeEntry() { var stopWatch = _log != null ? Stopwatch.StartNew() : null; try { - if (!_entry.CanUpdateAnalysis(Version, out var module, out var ast, out var currentAnalysis)) { - if (currentAnalysis is LibraryAnalysis) { - return; - } else if (ast == default) { - if (currentAnalysis == default) { - // Entry doesn't have ast yet. There should be at least one more session. - Cancel(); - } else { - Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); - } - } - - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + if (!CanUpdateAnalysis(_entry, null, Version, out var module, out var ast, out var currentAnalysis)) { return; } - var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(null, _entry, module, ast, Version); LogCompleted(null, module, stopWatch, startTime); @@ -327,44 +300,145 @@ private void AnalyzeEntry() { } } + private bool CanUpdateAnalysis( + PythonAnalyzerEntry entry, + IDependencyChainNode node, + int version, + out IPythonModule module, + out PythonAst ast, + out IDocumentAnalysis currentAnalysis) { + + if (!entry.CanUpdateAnalysis(version, out module, out ast, out currentAnalysis)) { + if (IsAnalyzedLibraryInLoop(node, currentAnalysis)) { + // Library analysis exists, don't analyze again + return false; + } + if (ast == default) { + if (currentAnalysis == default) { + // Entry doesn't have ast yet. There should be at least one more session. + Cancel(); + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet)."); + return false; + } + //Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); + return false; + } + + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}."); + return false; + } + return true; + } + private void AnalyzeEntry(IDependencyChainNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var walker = new ModuleWalker(_services, module, ast); - ast.Walk(walker); - + Debug.Assert(ast != null); + var analysis = DoAnalyzeEntry(node, module, ast, version); _analyzerCancellationToken.ThrowIfCancellationRequested(); + if (analysis != null) { + analyzable?.NotifyAnalysisComplete(analysis); + entry.TrySetAnalysis(analysis, version); + + if (module.ModuleType == ModuleType.User) { + var linterDiagnostics = _analyzer.LintModule(module); + _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); + } + } + } + + private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IPythonModule module, PythonAst ast, int version) { + var analysis = TryRestoreCachedAnalysis(node, module); + if (analysis != null) { + return analysis; + } + + var walker = new ModuleWalker(_services, module, ast, _analyzerCancellationToken); + ast.Walk(walker); walker.Complete(); - _analyzerCancellationToken.ThrowIfCancellationRequested(); + return CreateAnalysis(node, (IDocument)module, ast, version, walker); + } + private bool MarkNodeWalked(IDependencyChainNode node) { bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; } - if (!isCanceled) { node?.MarkWalked(); } + return isCanceled; + } - var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled); - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); + private IDocumentAnalysis TryRestoreCachedAnalysis(IDependencyChainNode node, IPythonModule module) { + var moduleType = module.ModuleType; + if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(module.Name, module.FilePath) == true) { + if (_moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, "Restored from database: ", module.Name); + } + var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty()); + gs.ReconstructVariables(); + MarkNodeWalked(node); + return analysis; + } else { + if (_log != null) { + _log.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name); + } + } + } + return null; + } + + private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { + var canHaveLibraryAnalysis = false; + + // Don't try to drop builtins; it causes issues elsewhere. + // We probably want the builtin module's AST and other info for evaluation. + switch (document.ModuleType) { + case ModuleType.Library: + case ModuleType.Compiled: + case ModuleType.CompiledBuiltin: + canHaveLibraryAnalysis = true; + break; + } + + var isCanceled = MarkNodeWalked(node); + var createLibraryAnalysis = !isCanceled && + node != null && + !node.HasMissingDependencies && + canHaveLibraryAnalysis && + !document.IsOpen && + node.HasOnlyWalkedDependencies && + node.IsValidVersion; - if (module.ModuleType == ModuleType.User) { - var linterDiagnostics = _analyzer.LintModule(module); - _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); + if (!createLibraryAnalysis) { + return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); } + + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); + document.SetAst(ast); + + var eval = new ExpressionEval(walker.Eval.Services, document, ast); + var analysis = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + + var dbs = _services.GetService(); + dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait(); + + return analysis; } + private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { if (_log != null) { var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; + var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); var message = node != null - ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms." - : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."; + ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {elapsed} ms." + : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {elapsed} ms."; _log.Log(TraceEventType.Verbose, message); } } @@ -385,37 +459,6 @@ private void LogException(IPythonModule module, Exception exception) { } } - private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { - var canHaveLibraryAnalysis = false; - - // Don't try to drop builtins; it causes issues elsewhere. - // We probably want the builtin module's AST and other info for evaluation. - switch (document.ModuleType) { - case ModuleType.Library: - case ModuleType.Stub: - case ModuleType.Compiled: - canHaveLibraryAnalysis = true; - break; - } - - var createLibraryAnalysis = !isCanceled && - node != null && - !node.HasMissingDependencies && - canHaveLibraryAnalysis && - !document.IsOpen && - node.HasOnlyWalkedDependencies && - node.IsValidVersion; - - if (!createLibraryAnalysis) { - return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); - } - - ast.Reduce(x => x is ImportStatement || x is FromImportStatement); - document.SetAst(ast); - var eval = new ExpressionEval(walker.Eval.Services, document, ast); - return new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); - } - private enum State { NotStarted = 0, Started = 1, diff --git a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs new file mode 100644 index 000000000..d084e623d --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs @@ -0,0 +1,308 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Analyzer { + internal sealed class StubMerger { + private readonly ExpressionEval _eval; + + public StubMerger(ExpressionEval eval) { + _eval = eval; + } + + /// + /// Merges data from stub with the data from the module. + /// + /// + /// Types are taken from the stub while location and documentation comes from + /// source so code navigation takes user to the source and not to the stub. + /// Stub data, such as class methods are augmented by methods from source + /// since stub is not guaranteed to be complete. + /// + public void MergeStub(IDocumentAnalysis stubAnalysis, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (_eval.Module.ModuleType == ModuleType.User || _eval.Module.ModuleType == ModuleType.Stub) { + return; + } + // No stub, no merge. + if (stubAnalysis.IsEmpty()) { + return; + } + // https://github.com/microsoft/python-language-server/issues/907 + Debug.Assert(!(stubAnalysis is EmptyAnalysis)); + + // Stub is the primary information source. Take types from the stub + // and replace source types by stub types. Transfer location and documentation + // from source members to the stub types. + TransferTypesFromStub(stubAnalysis, cancellationToken); + + UpdateVariables(); + } + + /// + /// Transfers types from stub to source while preserving documentation and location. + /// + /// + /// Stub is the primary information source. Take types from the stub + /// and replace source types by stub types. Transfer location and documentation + /// from source members to the stub types. + /// + private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationToken cancellationToken) { + foreach (var v in stubAnalysis.GlobalScope.Variables.ToArray()) { + cancellationToken.ThrowIfCancellationRequested(); + + var stubType = v.Value.GetPythonType(); + if (stubType.IsUnknown()) { + continue; + } + // If stub says 'Any' but we have better type, keep the current type. + if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { + continue; + } + + var sourceVar = _eval.GlobalScope.Variables[v.Name]; + var sourceType = sourceVar?.Value.GetPythonType(); + + if (sourceVar?.Source == VariableSource.Import && + sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { + // Keep imported types as they are defined in the library. For example, + // 'requests' imports NullHandler as 'from logging import NullHandler'. + // But 'requests' also declares NullHandler in its stub (but not in the main code) + // and that declaration does not have documentation or location. Therefore avoid + // taking types that are stub-only when similar type is imported from another + // module that also has a stub. + continue; + } + + TryReplaceMember(v, sourceType, stubType, cancellationToken); + } + } + + private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType stubType, CancellationToken cancellationToken) { + // If type does not exist in module, but exists in stub, declare it unless it is an import. + // If types are the classes, take class from the stub, then add missing members. + // Otherwise, replace type by one from the stub. + switch (sourceType) { + case null: + // Nothing in source, but there is type in the stub. Declare it. + if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) { + _eval.DeclareVariable(v.Name, v.Value, v.Source); + } + break; + + case PythonClassType sourceClass: + MergeClass(v, sourceClass, stubType, cancellationToken); + break; + + case IPythonModule _: + // We do not re-declare modules. + break; + + default: + var stubModule = stubType.DeclaringModule; + if (stubType is IPythonModule || stubModule.ModuleType == ModuleType.Builtins) { + // Modules members that are modules should remain as they are, i.e. os.path + // should remain library with its own stub attached. + break; + } + // We do not re-declaring variables that are imported. + if (v.Source == VariableSource.Declaration) { + TransferDocumentationAndLocation(sourceType, stubType); + // Re-declare variable with the data from the stub. + var source = _eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source; + _eval.DeclareVariable(v.Name, v.Value, source); + } + + break; + } + } + + private void MergeClass(IVariable v, IPythonClassType sourceClass, IPythonType stubType, CancellationToken cancellationToken) { + // Transfer documentation first so we get class documentation + // that comes from the class definition win over one that may + // come from __init__ during the member merge below. + TransferDocumentationAndLocation(sourceClass, stubType); + + // Replace the class entirely since stub members may use generic types + // and the class definition is important. We transfer missing members + // from the original class to the stub. + _eval.DeclareVariable(v.Name, v.Value, v.Source); + + // First pass: go through source class members and pick those + // that are not present in the stub class. + foreach (var name in sourceClass.GetMemberNames().ToArray()) { + cancellationToken.ThrowIfCancellationRequested(); + + var sourceMember = sourceClass.GetMember(name); + if (sourceMember.IsUnknown()) { + continue; // Do not add unknowns to the stub. + } + var sourceMemberType = sourceMember?.GetPythonType(); + if (sourceMemberType is IPythonClassMember cm && cm.DeclaringType != sourceClass) { + continue; // Only take members from this class and not from bases. + } + if (!IsFromThisModuleOrSubmodules(sourceMemberType)) { + continue; // Member does not come from module or its submodules. + } + + var stubMember = stubType.GetMember(name); + var stubMemberType = stubMember.GetPythonType(); + + // Get documentation from the current type, if any, since stubs + // typically do not contain documentation while scraped code does. + TransferDocumentationAndLocation(sourceMemberType, stubMemberType); + + // If stub says 'Any' but we have better type, use member from the original class. + if (stubMember != null && !(stubType.DeclaringModule is TypingModule && stubType.Name == "Any")) { + continue; // Stub already have the member, don't replace. + } + + (stubType as PythonType)?.AddMember(name, stubMember, overwrite: true); + } + + // Second pass: go through stub class members and if they don't have documentation + // or location, check if source class has same member and fetch it from there. + // The reason is that in the stub sometimes members are specified all in one + // class while in source they may come from bases. Example: datetime. + foreach (var name in stubType.GetMemberNames().ToArray()) { + cancellationToken.ThrowIfCancellationRequested(); + + var stubMember = stubType.GetMember(name); + if (stubMember.IsUnknown()) { + continue; + } + var stubMemberType = stubMember.GetPythonType(); + if (stubMemberType is IPythonClassMember cm && cm.DeclaringType != stubType) { + continue; // Only take members from this class and not from bases. + } + + var sourceMember = sourceClass.GetMember(name); + if (sourceMember.IsUnknown()) { + continue; + } + + var sourceMemberType = sourceMember.GetPythonType(); + if (sourceMemberType.Location.IsValid && !stubMemberType.Location.IsValid) { + TransferDocumentationAndLocation(sourceMemberType, stubMemberType); + } + } + } + + private void UpdateVariables() { + // Second pass: if we replaced any classes by new from the stub, we need to update + // variables that may still be holding old content. For example, ctypes + // declares 'c_voidp = c_void_p' so when we replace 'class c_void_p' + // by class from the stub, we need to go and update 'c_voidp' variable. + foreach (var v in _eval.GlobalScope.Variables.ToArray()) { + var variableType = v.Value.GetPythonType(); + if (!IsFromThisModuleOrSubmodules(variableType)) { + continue; + } + // Check if type that the variable references actually declared here. + var typeInScope = _eval.GlobalScope.Variables[variableType.Name].GetPythonType(); + if (typeInScope == null || variableType == typeInScope) { + continue; + } + + if (v.Value == variableType) { + _eval.DeclareVariable(v.Name, typeInScope, v.Source); + } else if (v.Value is IPythonInstance) { + _eval.DeclareVariable(v.Name, new PythonInstance(typeInScope), v.Source); + } + } + } + + private void TransferDocumentationAndLocation(IPythonType sourceType, IPythonType stubType) { + if (sourceType.IsUnknown() || sourceType.DeclaringModule.ModuleType == ModuleType.Builtins || + stubType.IsUnknown() || stubType.DeclaringModule.ModuleType == ModuleType.Builtins) { + return; // Do not transfer location of unknowns or builtins + } + + // Stub may be one for multiple modules - when module consists of several + // submodules, there is typically only one stub for the main module. + // Types from 'unittest.case' (library) are stubbed in 'unittest' stub. + if (!IsFromThisModuleOrSubmodules(sourceType)) { + return; // Do not change unrelated types. + } + + // Destination must be from this module stub and not from other modules. + // Consider that 'email.headregistry' stub has DataHeader declaring 'datetime' + // property of type 'datetime' from 'datetime' module. We don't want to modify + // datetime type and change it's location to 'email.headregistry'. + if(stubType.DeclaringModule.ModuleType != ModuleType.Stub || stubType.DeclaringModule != _eval.Module.Stub) { + return; + } + + // Documentation and location are always get transferred from module type + // to the stub type and never the other way around. This makes sure that + // we show documentation from the original module and goto definition + // navigates to the module source and not to the stub. + if (sourceType != stubType && sourceType is PythonType src && stubType is PythonType dst) { + // If type is a class, then doc can either come from class definition node of from __init__. + // If class has doc from the class definition, don't stomp on it. + if (src is PythonClassType srcClass && dst is PythonClassType dstClass) { + // Higher lever source wins + if (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Class || + (srcClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Init && dstClass.DocumentationSource == PythonClassType.ClassDocumentationSource.Base)) { + dstClass.SetDocumentation(srcClass.Documentation); + } + } else { + // Sometimes destination (stub type) already has documentation. This happens when stub type + // is used to augment more than one type. For example, in threading module RLock stub class + // replaces both RLock function and _RLock class making 'factory' function RLock to look + // like a class constructor. Effectively a single stub type is used for more than one type + // in the source and two source types may have different documentation. Thus transferring doc + // from one source type affects documentation of another type. It may be better to clone stub + // type and separate instances for separate source type, but for now we'll just avoid stomping + // on the existing documentation. + if (string.IsNullOrEmpty(dst.Documentation)) { + var srcDocumentation = src.Documentation; + if (!string.IsNullOrEmpty(srcDocumentation)) { + dst.SetDocumentation(srcDocumentation); + } + } + } + + if (src.Location.IsValid) { + dst.Location = src.Location; + } + } + } + + /// + /// Determines if type comes from module that is part of this package. + /// + /// + /// Single stub file typically applies to all modules while types within + /// the package come from multiple modules. We need to determine if stub + /// does match the type module so we don't accidentally modify documentation + /// or location of unrelated types such as coming from the base object type. + /// + private bool IsFromThisModuleOrSubmodules(IPythonType type) { + var thisModule = _eval.Module; + var typeModule = type.DeclaringModule; + var typeMainModuleName = typeModule.Name.Split('.').FirstOrDefault(); + return typeModule.Equals(thisModule) || typeMainModuleName == thisModule.Name; + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index 4f67b7660..c563428c5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -54,7 +54,6 @@ public void EvaluateClass() { var bases = ProcessBases(); _class.SetBases(bases, Eval.CurrentScope); - _class.DecideGeneric(); // Declare __class__ variable in the scope. Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); ProcessClassBody(); @@ -110,16 +109,18 @@ private void ProcessClassBody() { } private IEnumerable ProcessBases() { - var bases = new List(); - foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name))) { - if (IsValidBase(a)) { - TryAddBase(bases, a); - } else { - ReportInvalidBase(a); + // Base types must be evaluated in outer scope + using (Eval.OpenScope(Eval.CurrentScope.OuterScope)) { + var bases = new List(); + foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name))) { + if (IsValidBase(a)) { + TryAddBase(bases, a); + } else { + ReportInvalidBase(a); + } } + return bases; } - - return bases; } private bool IsValidBase(Arg a) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index c323d7786..f50d4f705 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -64,10 +64,16 @@ public override void Evaluate() { if (ctor || returnType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); - // For libraries remove declared local function variables to free up some memory. + // For libraries remove declared local function variables to free up some memory + // unless function has inner classes or functions. var optionsProvider = Eval.Services.GetService(); - if (Module.ModuleType != ModuleType.User && optionsProvider?.Options.KeepLibraryLocalVariables != true) { - ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + if (Module.ModuleType != ModuleType.User && + optionsProvider?.Options.KeepLibraryLocalVariables != true && + Eval.CurrentScope.Variables.All( + v => v.GetPythonType() == null && + v.GetPythonType() == null) + ) { + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); } } } @@ -79,6 +85,7 @@ private IPythonType TryDetermineReturnValue() { if (!annotationType.IsUnknown()) { // Annotations are typically types while actually functions return // instances unless specifically annotated to a type such as Type[T]. + // TODO: try constructing argument set from types. Consider Tuple[_T1, _T2] where _T1 = TypeVar('_T1', str, bytes) var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.WithoutContext); // If instance could not be created, such as when return type is List[T] and // type of T is not yet known, just use the type. @@ -118,6 +125,7 @@ public override bool Walk(AssignmentStatement node) { public override bool Walk(ReturnStatement node) { var value = Eval.GetValueFromExpression(node.Expression); if (value != null) { + // although technically legal, __init__ in a constructor should not have a not-none return value if (_function.IsDunderInit() && !value.IsOfType(BuiltinTypeId.NoneType)) { Eval.ReportDiagnostics(Module.Uri, new DiagnosticsEntry( diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index f1402a4bc..b08c0f07e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; @@ -29,7 +30,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Symbols { /// so the symbol table can resolve references on demand. /// internal sealed class SymbolCollector : PythonWalker { - private readonly Dictionary _typeMap = new Dictionary(); + private readonly Dictionary _typeMap = new Dictionary(); private readonly Stack _scopes = new Stack(); private readonly ModuleSymbolTable _table; private readonly ExpressionEval _eval; @@ -47,7 +48,7 @@ private SymbolCollector(ModuleSymbolTable table, ExpressionEval eval) { private void Walk() => _eval.Ast.Walk(this); public override bool Walk(IfStatement node) - => node.WalkIfWithSystemConditions(this, _eval.Ast.LanguageVersion, _eval.Services.GetService().IsWindows); + => node.WalkIfWithSystemConditions(this, _eval.Ast.LanguageVersion, _eval.Services.GetService()); public override bool Walk(ClassDefinition cd) { if (IsDeprecated(cd)) { @@ -93,9 +94,16 @@ public override void PostWalk(FunctionDefinition fd) { } private PythonClassType CreateClass(ClassDefinition cd) { - var cls = new PythonClassType(cd, _eval.GetLocationOfName(cd), + PythonType declaringType = null; + if(!(cd.Parent is PythonAst)) { + Debug.Assert(_typeMap.ContainsKey(cd.Parent)); + _typeMap.TryGetValue(cd.Parent, out declaringType); + } + var cls = new PythonClassType(cd, declaringType, _eval.GetLocationOfName(cd), _eval.SuppressBuiltinLookup ? BuiltinTypeId.Unknown : BuiltinTypeId.Type); _typeMap[cd] = cls; + + declaringType?.AddMember(cls.Name, cls, overwrite: true); return cls; } @@ -107,14 +115,16 @@ private void AddFunctionOrProperty(FunctionDefinition fd) { } } - private void AddFunction(FunctionDefinition fd, IPythonType declaringType) { - if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonFunctionType existing)) { - existing = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd)); + private void AddFunction(FunctionDefinition fd, PythonType declaringType) { + if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonFunctionType f)) { + f = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd)); // The variable is transient (non-user declared) hence it does not have location. // Function type is tracking locations for references and renaming. - _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); + _eval.DeclareVariable(fd.Name, f, VariableSource.Declaration); + _typeMap[fd] = f; + declaringType?.AddMember(f.Name, f, overwrite: true); } - AddOverload(fd, existing, o => existing.AddOverload(o)); + AddOverload(fd, f, o => f.AddOverload(o)); } private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { @@ -152,7 +162,7 @@ private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { return null; } - private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) { + private bool TryAddProperty(FunctionDefinition node, PythonType declaringType) { // We can't add a property to an unknown type. Fallback to a regular function for now. // TOOD: Decouple declaring types from the property. if (declaringType.IsUnknown()) { @@ -175,14 +185,16 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) return false; } - private void AddProperty(FunctionDefinition fd, IPythonType declaringType, bool isAbstract) { - if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType existing)) { - existing = new PythonPropertyType(fd, _eval.GetLocationOfName(fd), declaringType, isAbstract); + private void AddProperty(FunctionDefinition fd, PythonType declaringType, bool isAbstract) { + if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType p)) { + p = new PythonPropertyType(fd, _eval.GetLocationOfName(fd), declaringType, isAbstract); // The variable is transient (non-user declared) hence it does not have location. // Property type is tracking locations for references and renaming. - _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); + _eval.DeclareVariable(fd.Name, p, VariableSource.Declaration); + _typeMap[fd] = p; + declaringType?.AddMember(p.Name, p, overwrite: true); } - AddOverload(fd, existing, o => existing.AddOverload(o)); + AddOverload(fd, p, o => p.AddOverload(o)); } private IMember GetMemberFromStub(string name) { diff --git a/src/Analysis/Ast/Impl/Caching/CacheFolders.cs b/src/Analysis/Ast/Impl/Caching/CacheFolders.cs index 3ee3508d0..414dec1e3 100644 --- a/src/Analysis/Ast/Impl/Caching/CacheFolders.cs +++ b/src/Analysis/Ast/Impl/Caching/CacheFolders.cs @@ -19,26 +19,27 @@ using System.Security.Cryptography; using System.Text; using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.OS; namespace Microsoft.Python.Analysis.Caching { - internal static class CacheFolders { - public static string GetCacheFilePath(string root, string moduleName, string content, IFileSystem fs) { - // Folder for all module versions and variants - // {root}/module_name/content_hash.pyi - var folder = Path.Combine(root, moduleName); - - var filePath = Path.Combine(root, folder, $"{FileNameFromContent(content)}.pyi"); - if (fs.StringComparison == StringComparison.Ordinal) { - filePath = filePath.ToLowerInvariant(); - } + internal sealed class CacheFolderService: ICacheFolderService { + public CacheFolderService(IServiceContainer services, string cacheRootFolder) { + CacheFolder = cacheRootFolder ?? GetCacheFolder(services); + } + + public string CacheFolder { get; } - return filePath; + public string GetFileNameFromContent(string content) { + // File name depends on the content so we can distinguish between different versions. + using (var hash = SHA256.Create()) { + return Convert + .ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content))) + .Replace('/', '_').Replace('+', '-'); + } } - public static string GetCacheFolder(IServiceContainer services) { + private static string GetCacheFolder(IServiceContainer services) { var platform = services.GetService(); var logger = services.GetService(); @@ -95,17 +96,6 @@ public static string GetCacheFolder(IServiceContainer services) { return cachePath; } - public static string FileNameFromContent(string content) { - // File name depends on the content so we can distinguish between different versions. - var hash = SHA256.Create(); - return Convert - .ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content))) - .Replace('/', '_').Replace('+', '-'); - } - - public static string GetAnalysisCacheFilePath(string analysisRootFolder, string moduleName, string content, IFileSystem fs) - => GetCacheFilePath(analysisRootFolder, moduleName, content, fs); - private static bool CheckPathRooted(string varName, string path, ILogger logger) { if (!string.IsNullOrWhiteSpace(path) && Path.IsPathRooted(path)) { return true; diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs new file mode 100644 index 000000000..acbe31f5d --- /dev/null +++ b/src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs @@ -0,0 +1,21 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching { + public interface ICacheFolderService { + string CacheFolder { get; } + string GetFileNameFromContent(string content); + } +} diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs new file mode 100644 index 000000000..9477a952f --- /dev/null +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -0,0 +1,64 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Caching { + /// + /// Represents global scope that has been restored from + /// the database but has not been fully populated yet. + /// Used to attach to analysis so variables can be + /// accessed during classes and methods restoration. + /// + internal interface IRestoredGlobalScope : IGlobalScope { + void ReconstructVariables(); + } + + internal interface IModuleDatabaseService { + /// + /// Creates global scope from module persistent state. + /// Global scope is then can be used to construct module analysis. + /// + /// Python module to restore analysis for. + /// Python module global scope. + bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs); + + /// + /// Retrieves dependencies from the module persistent state. + /// + /// Python module to restore analysis for. + /// Python module dependency provider. + bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp); + + /// + /// Writes module data to the database. + /// + Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default); + + /// + /// Determines if module analysis exists in the storage. + /// + bool ModuleExistsInStorage(string moduleName, string filePath); + + /// + /// Clear cached data. + /// + void Clear(); + } +} diff --git a/src/Analysis/Ast/Impl/Caching/StubCache.cs b/src/Analysis/Ast/Impl/Caching/StubCache.cs index 170bebd21..58081d146 100644 --- a/src/Analysis/Ast/Impl/Caching/StubCache.cs +++ b/src/Analysis/Ast/Impl/Caching/StubCache.cs @@ -27,13 +27,14 @@ internal sealed class StubCache : IStubCache { private readonly IFileSystem _fs; private readonly ILogger _log; + private readonly ICacheFolderService _cfs; - public StubCache(IServiceContainer services, string cacheRootFolder = null) { + public StubCache(IServiceContainer services) { _fs = services.GetService(); _log = services.GetService(); - cacheRootFolder = cacheRootFolder ?? CacheFolders.GetCacheFolder(services); - StubCacheFolder = Path.Combine(cacheRootFolder, $"stubs.v{_stubCacheFormatVersion}"); + _cfs = services.GetService(); + StubCacheFolder = Path.Combine(_cfs.CacheFolder, $"stubs.v{_stubCacheFormatVersion}"); } public string StubCacheFolder { get; } @@ -58,7 +59,7 @@ public string GetCacheFilePath(string filePath) { dir = dir.ToLowerInvariant(); } - var dirHash = CacheFolders.FileNameFromContent(dir); + var dirHash = _cfs.GetFileNameFromContent(dir); var stubFile = Path.Combine(StubCacheFolder, Path.Combine(dirHash, name)); return Path.ChangeExtension(stubFile, ".pyi"); } diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs index 36c4eb180..dcf73ec35 100644 --- a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -14,6 +14,25 @@ // permissions and limitations under the License. namespace Microsoft.Python.Analysis { + public enum AnalysisCachingLevel { + /// + /// No caching of analysis. + /// + None, + + /// + /// Cache analysis results of system (language) modules. + /// Do not cache user-installed modules or site-packages. + /// + System, + + /// + /// Full caching, includes system and library modules. + /// Does not enable caching of user code analysis. + /// + Library + } + public class AnalysisOptions { public bool LintingEnabled { get; set; } @@ -29,5 +48,10 @@ public class AnalysisOptions { /// improve performance when library code has to be re-analyzed. /// public bool KeepLibraryAst { get; set; } + + /// + /// Defines level of caching analysis engine will maintain. + /// + public AnalysisCachingLevel AnalysisCachingLevel { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs new file mode 100644 index 000000000..586e314c1 --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs @@ -0,0 +1,77 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Dependencies { + internal sealed class DependencyCollector { + private readonly IPythonModule _module; + private readonly bool _isTypeshed; + private readonly IModuleManagement _moduleResolution; + private readonly PathResolverSnapshot _pathResolver; + + public ISet Dependencies { get; } = new HashSet(); + + public DependencyCollector(IPythonModule module, bool? isTypeShed = null) { + _module = module; + _isTypeshed = isTypeShed ?? module is StubPythonModule stub && stub.IsTypeshed; + _moduleResolution = module.Interpreter.ModuleResolution; + _pathResolver = _isTypeshed + ? module.Interpreter.TypeshedResolution.CurrentPathResolver + : _moduleResolution.CurrentPathResolver; + + if (module.Stub != null) { + Dependencies.Add(new AnalysisModuleKey(module.Stub)); + } + } + + public void AddImport(IReadOnlyList importNames, bool forceAbsolute) { + var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); + HandleSearchResults(imports); + } + + public void AddFromImport(IEnumerable importNames, IEnumerable memberNames, int dotCount, bool forceAbsolute) { + var imports = _pathResolver.FindImports(_module.FilePath, importNames, dotCount, forceAbsolute); + HandleSearchResults(imports); + if (imports is IImportChildrenSource childrenSource) { + foreach (var name in memberNames) { + if (childrenSource.TryGetChildImport(name, out var childImport)) { + HandleSearchResults(childImport); + } + } + } + } + + private void HandleSearchResults(IImportSearchResult searchResult) { + switch (searchResult) { + case ModuleImport moduleImport when !Ignore(_moduleResolution, moduleImport.FullName, moduleImport.ModulePath): + Dependencies.Add(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed)); + return; + case PossibleModuleImport possibleModuleImport when !Ignore(_moduleResolution, possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath): + Dependencies.Add(new AnalysisModuleKey(possibleModuleImport.PrecedingModuleFullName, possibleModuleImport.PrecedingModulePath, _isTypeshed)); + return; + default: + return; + } + } + private static bool Ignore(IModuleManagement moduleResolution, string fullName, string modulePath) + => moduleResolution.BuiltinModuleName.EqualsOrdinal(fullName) || moduleResolution.IsSpecializedModule(fullName, modulePath); + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs new file mode 100644 index 000000000..2f2270251 --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs @@ -0,0 +1,55 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Collections; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Dependencies { + internal sealed class DependencyWalker : PythonWalker { + private readonly DependencyCollector _dependencyCollector; + + public ISet Dependencies => _dependencyCollector.Dependencies; + + public DependencyWalker(IPythonModule module, PythonAst ast = null) { + _dependencyCollector = new DependencyCollector(module); + ast = ast ?? module.GetAst(); + ast.Walk(this); + } + + public override bool Walk(ImportStatement import) { + var forceAbsolute = import.ForceAbsolute; + foreach (var moduleName in import.Names) { + var importNames = ImmutableArray.Empty; + foreach (var nameExpression in moduleName.Names) { + importNames = importNames.Add(nameExpression.Name); + _dependencyCollector.AddImport(importNames, forceAbsolute); + } + } + return false; + } + + public override bool Walk(FromImportStatement fromImport) { + var rootNames = fromImport.Root.Names.Select(n => n.Name); + var memberNames = fromImport.Names.Select(n => n.Name); + var dotCount = fromImport.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0; + _dependencyCollector.AddFromImport(rootNames, memberNames, dotCount, fromImport.ForceAbsolute); + return false; + } + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs new file mode 100644 index 000000000..3a0c8ba0d --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyProvider.cs @@ -0,0 +1,29 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Dependencies { + /// + /// Implements provider that can supply list of imports to the dependency analysis. + /// Regular modules provide dependency from the AST, persistent/database modules + /// provide dependencies from their models. + /// + internal interface IDependencyProvider { + ISet GetDependencies(PythonAst ast); + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index 02fdfbdfd..3816803e3 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Analysis.Dependencies { diff --git a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs index dcbbc12bb..21a2bdfb8 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs +++ b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs @@ -50,7 +50,7 @@ public void Update(IEnumerable changes) { // Every change may change where the lines end so in order // to correctly determine line/offsets we must re-split buffer // into lines after each change. - var lineLoc = GetNewLineLications().ToArray(); + var lineLoc = GetNewLineLocations().ToArray(); if (change.ReplaceAllText) { _sb = new StringBuilder(change.InsertedText); @@ -72,32 +72,35 @@ public void Update(IEnumerable changes) { } } - public IEnumerable GetNewLineLications() { - _sb = _sb ?? new StringBuilder(_content); // for tests + public IEnumerable GetNewLineLocations() { + lock (_lock) { + _sb = _sb ?? new StringBuilder(_content); // for tests - if (_sb.Length == 0) { - yield return new NewLineLocation(0, NewLineKind.None); - } + if (_sb.Length == 0) { + yield return new NewLineLocation(0, NewLineKind.None); + } - for (var i = 0; i < _sb.Length; i++) { - var ch = _sb[i]; - var nextCh = i < _sb.Length - 1 ? _sb[i + 1] : '\0'; - switch (ch) { - case '\r' when nextCh == '\n': - i++; - yield return new NewLineLocation(i + 1, NewLineKind.CarriageReturnLineFeed); - break; - case '\n': - yield return new NewLineLocation(i + 1, NewLineKind.LineFeed); - break; - case '\r': - yield return new NewLineLocation(i + 1, NewLineKind.CarriageReturn); - break; - default: - if (i == _sb.Length - 1) { - yield return new NewLineLocation(i + 1, NewLineKind.None); - } - break; + for (var i = 0; i < _sb.Length; i++) { + var ch = _sb[i]; + var nextCh = i < _sb.Length - 1 ? _sb[i + 1] : '\0'; + switch (ch) { + case '\r' when nextCh == '\n': + i++; + yield return new NewLineLocation(i + 1, NewLineKind.CarriageReturnLineFeed); + break; + case '\n': + yield return new NewLineLocation(i + 1, NewLineKind.LineFeed); + break; + case '\r': + yield return new NewLineLocation(i + 1, NewLineKind.CarriageReturn); + break; + default: + if (i == _sb.Length - 1) { + yield return new NewLineLocation(i + 1, NewLineKind.None); + } + + break; + } } } } diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 1f802edd8..810632abc 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -241,10 +241,10 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) { IDocument document; switch (mco.ModuleType) { case ModuleType.Compiled when TryAddModulePath(mco): - document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, _services); + document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsPersistent, _services); break; case ModuleType.CompiledBuiltin: - document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, _services); + document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, mco.IsPersistent, _services); break; case ModuleType.User: TryAddModulePath(mco); diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 555a21567..2ec61b377 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -22,6 +22,8 @@ namespace Microsoft.Python.Analysis { public static class AnalysisExtensions { + public static bool IsEmpty(this IDocumentAnalysis analysis) => analysis == null || analysis is EmptyAnalysis; + public static IScope FindScope(this IDocumentAnalysis analysis, SourceLocation location) => analysis.GlobalScope.FindScope(analysis.Document, location); @@ -58,7 +60,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana // 'type()' in code is a function call, not a type class instantiation. if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty); - f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); + f.AddOverload(new PythonFunctionOverload(f, new Location(analysis.Document))); analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); } return f; diff --git a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs index f45e95409..6699f54d8 100644 --- a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs @@ -13,9 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. - using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography.X509Certificates; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; @@ -33,8 +33,8 @@ public static IReadOnlyList> Arguments(this IArgument public static T Argument(this IArgumentSet args, int index) where T : class => args.Arguments[index].Value as T; - public static T GetArgumentValue(this IArgumentSet args, string name) where T : class { - var value = args.Arguments.FirstOrDefault(a => name.Equals(a.Name))?.Value; + public static T GetArgumentValue(this IArgumentSet args, string name, bool excludeDefault = true) where T : class { + var value = args.Arguments.FirstOrDefault(a => name.Equals(a.Name) && !(excludeDefault && a.ValueIsDefault))?.Value; if (value == null && args.DictionaryArgument?.Arguments != null && args.DictionaryArgument.Arguments.TryGetValue(name, out var m)) { return m as T; } @@ -63,7 +63,7 @@ internal static void DeclareParametersInScope(this IArgumentSet args, Expression } if (args.ListArgument != null && !string.IsNullOrEmpty(args.ListArgument.Name)) { - var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.Interpreter, false); + var type = new PythonCollectionType(BuiltinTypeId.List, eval.BuiltinsModule, false); var list = new PythonCollection(type, args.ListArgument.Values); eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location); } diff --git a/src/Analysis/Ast/Impl/Extensions/IfStatementExtensions.cs b/src/Analysis/Ast/Impl/Extensions/IfStatementExtensions.cs index 43a8e6869..02efaa287 100644 --- a/src/Analysis/Ast/Impl/Extensions/IfStatementExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/IfStatementExtensions.cs @@ -90,5 +90,24 @@ public static ConditionTestResult TryHandleOsPath(this IfStatementTest test, boo } return ConditionTestResult.Unrecognized; } + + public static ConditionTestResult TryHandleBigLittleEndian(this IfStatementTest test, bool isLittleEndian) { + if (test.Test is BinaryExpression cmp && + cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "byteorder" && + cmp.Right is ConstantExpression cex && cex.GetStringValue() is string s) { + switch (cmp.Operator) { + case PythonOperator.Equals when s == "little" && isLittleEndian: + return ConditionTestResult.WalkBody; + case PythonOperator.Equals when s == "big" && !isLittleEndian: + return ConditionTestResult.WalkBody; + case PythonOperator.NotEquals when s == "little" && !isLittleEndian: + return ConditionTestResult.WalkBody; + case PythonOperator.NotEquals when s == "big" && isLittleEndian: + return ConditionTestResult.WalkBody; + } + return ConditionTestResult.DontWalkBody; + } + return ConditionTestResult.Unrecognized; + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index 508ee9641..aa82e938a 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -26,7 +26,6 @@ public static LocationInfo GetLocation(this Node node, IExpressionEvaluator eval return GetLocation(node, eval.Ast, eval.Module); } - public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysis) { if (node == null || node.StartIndex >= node.EndIndex) { return LocationInfo.Empty; @@ -35,10 +34,10 @@ public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysi return GetLocation(node, analysis.Ast, analysis.Document); } - private static LocationInfo GetLocation(Node node, PythonAst ast, IPythonFile pythonFile) { + private static LocationInfo GetLocation(Node node, PythonAst ast, IPythonModule module) { var start = node.GetStart(ast); var end = node.GetEnd(ast); - return new LocationInfo(pythonFile.FilePath, pythonFile.Uri, start.Line, start.Column, end.Line, end.Column); + return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); } public static Expression RemoveParenthesis(this Expression e) { diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 7ee3f4a85..d8705f253 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -61,8 +61,8 @@ public static bool IsPrivateMember(this IPythonClassType cls, string memberName) /// /// Gets specific type for the given generic type parameter, resolving bounds as well /// - public static bool GetSpecificType(this IPythonClassType cls, IGenericTypeParameter param, out IPythonType specificType) { - cls.GenericParameters.TryGetValue(param, out specificType); + public static bool GetSpecificType(this IPythonClassType cls, string paramName, out IPythonType specificType) { + cls.GenericParameters.TryGetValue(paramName, out specificType); // If type has not been found, check if the type parameter has an upper bound and use that if (specificType is IGenericTypeParameter gtp && gtp.Bound != null) { specificType = gtp.Bound; diff --git a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs index b1914d9c0..560d46332 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonFunctionExtensions.cs @@ -13,19 +13,23 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -namespace Microsoft.Python.Analysis.Extensions { +namespace Microsoft.Python.Analysis { public static class PythonFunctionExtensions { - public static bool IsUnbound(this IPythonFunctionType f) + public static bool IsUnbound(this IPythonFunctionType f) => f.DeclaringType != null && f.MemberType == PythonMemberType.Function; - public static bool IsBound(this IPythonFunctionType f) + public static bool IsBound(this IPythonFunctionType f) => f.DeclaringType != null && f.MemberType == PythonMemberType.Method; + public static bool IsLambda(this IPythonFunctionType f) => f.Name == ""; + public static bool HasClassFirstArgument(this IPythonClassMember m) => (m is IPythonFunctionType f && !f.IsStatic && (f.IsClassMethod || f.IsBound())) || (m is IPythonPropertyType prop); @@ -34,5 +38,16 @@ public static IScope GetScope(this IPythonFunctionType f) { IScope gs = f.DeclaringModule.GlobalScope; return gs?.TraverseBreadthFirst(s => s.Children).FirstOrDefault(s => s.Node == f.FunctionDefinition); } + + public static string GetQualifiedName(this IPythonClassMember cm, string baseName = null) { + var s = new Stack(); + s.Push(baseName ?? cm.Name); + for (var p = cm.DeclaringType as IPythonClassMember; p != null; p = p.DeclaringType as IPythonClassMember) { + s.Push(p.Name); + } + return cm.DeclaringModule.ModuleType == ModuleType.Builtins + ? string.Join(".", s) + : $"{cm.DeclaringModule.QualifiedName}:{string.Join(".", s)}"; + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index ca03c97d8..caa2b5b34 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -21,7 +21,7 @@ namespace Microsoft.Python.Analysis { public static class PythonModuleExtensions { internal static PythonAst GetAst(this IPythonModule module) - => (PythonAst)((IAstNodeContainer)module).GetAstNode(module); + => (PythonAst)(module as IAstNodeContainer)?.GetAstNode(module); internal static void SetAst(this IPythonModule module, PythonAst ast) { var contained = (IAstNodeContainer)module; diff --git a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs index 1075dc642..75a20a039 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs @@ -15,7 +15,6 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; namespace Microsoft.Python.Analysis { public static class PythonTypeExtensions { @@ -27,5 +26,9 @@ public static bool IsGenericParameter(this IPythonType value) public static bool IsGeneric(this IPythonType value) => value is IGenericTypeParameter || (value is IGenericType gt && gt.IsGeneric); + + public static string GetQualifiedName(this IPythonType t) => $"{t.DeclaringModule.Name}:{t.Name}"; + + internal static IPythonType ToBound(this IPythonType t) => t is PythonFunctionType.PythonUnboundMethod m ? m.Function : t; } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonWalkerExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonWalkerExtensions.cs index affefbf2d..f3420b46f 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonWalkerExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonWalkerExtensions.cs @@ -13,21 +13,25 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Core.OS; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { public static class PythonWalkerExtensions { - public static bool WalkIfWithSystemConditions(this IfStatement node, PythonWalker walker, PythonLanguageVersion languageVersion, bool isWindows) { + public static bool WalkIfWithSystemConditions(this IfStatement node, PythonWalker walker, PythonLanguageVersion languageVersion, IOSPlatform platform) { // System version, platform and os.path specializations var executeElse = false; foreach (var test in node.Tests) { var result = test.TryHandleSysVersionInfo(languageVersion); if (result == ConditionTestResult.Unrecognized) { - result = test.TryHandleSysPlatform(isWindows); + result = test.TryHandleSysPlatform(platform.IsWindows); if (result == ConditionTestResult.Unrecognized) { - result = test.TryHandleOsPath(isWindows); + result = test.TryHandleOsPath(platform.IsWindows); + if (result == ConditionTestResult.Unrecognized) { + result = test.TryHandleBigLittleEndian(platform.IsLittleEndian); + } } } diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 095df7033..72988f5a2 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -35,7 +35,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth private IPythonType _boolType; public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services) - : base(moduleName, ModuleType.Builtins, filePath, null, services) { } // TODO: builtins stub + : base(moduleName, ModuleType.Builtins, filePath, null, false, services) { } // TODO: builtins stub & persistence public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name); diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs index e020e7b26..500e4dbca 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents compiled module that is built into the language. /// internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule { - public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services) - : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, services) { } + public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, bool isPersistent, IServiceContainer services) + : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, isPersistent, services) { } protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sm) ? null : new [] { "-W", "ignore", "-B", "-E", sm, "-u8", Name }; diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index 913a872d4..662b25d30 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -26,8 +26,8 @@ namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache; - public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, IServiceContainer services) - : base(moduleName, filePath, moduleType, stub, services) { } + public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isPersistent, IServiceContainer services) + : base(moduleName, filePath, moduleType, stub, isPersistent, services) { } public override string Documentation => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty; @@ -93,7 +93,7 @@ private string ScrapeModule() { try { using (var process = new Process()) { process.StartInfo = startInfo; - process.ErrorDataReceived += new DataReceivedEventHandler((s, e) => { }); + process.ErrorDataReceived += (s, e) => { }; process.Start(); process.BeginErrorReadLine(); diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index 6d64bfc5e..1f2b0b936 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -55,15 +55,21 @@ public interface IModuleManagement : IModuleResolution { /// content is loaded and analyzed only for class/functions definitions /// so the original documentation can be extracted. /// - /// Module to specialize. + /// Module to specialize. /// Specialized module constructor. - /// Original (library) module loaded as stub, if any. - IPythonModule SpecializeModule(string name, Func specializationConstructor); + /// Replace existing loaded module, if any. + /// Specialized module. + IPythonModule SpecializeModule(string fullName, Func specializationConstructor, bool replaceExisting = false); /// - /// Returns specialized module, if any. + /// Returns specialized module, if any. Will attempt to load module from persistent state. /// - IPythonModule GetSpecializedModule(string name); + IPythonModule GetSpecializedModule(string fullName, bool allowCreation = false, string modulePath = null); + + /// + /// Determines of module is specialized or exists in the database. + /// + bool IsSpecializedModule(string fullName, string modulePath = null); /// /// Root directory of the path resolver. @@ -75,6 +81,11 @@ public interface IModuleManagement : IModuleResolution { /// ImmutableArray InterpreterPaths { get; } + /// + /// Interpreter paths with additional classification. + /// + ImmutableArray LibraryPaths { get; } + bool SetUserConfiguredPaths(ImmutableArray paths); } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs index b49e555dc..f3b1058c8 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs @@ -47,5 +47,10 @@ public sealed class ModuleCreationOptions { /// Module stub, if any. /// public IPythonModule Stub { get; set; } + + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs index a18205271..dba391b96 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs @@ -64,5 +64,6 @@ public enum ModuleType { public static class ModuleTypeExtensions { public static bool IsNonUserFile(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Stub; public static bool IsCompiled(this ModuleType type) => type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin; + public static bool CanBeCached(this ModuleType type) => type == ModuleType.Library || type == ModuleType.Compiled || type == ModuleType.CompiledBuiltin; } } diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs new file mode 100644 index 000000000..1e0e4752a --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Modules { + internal sealed class DependencyProvider: IDependencyProvider { + private readonly IPythonModule _module; + private readonly IModuleDatabaseService _dbService; + + public static IDependencyProvider Empty { get; } = new EmptyDependencyProvider(); + + public DependencyProvider(IPythonModule module, IServiceContainer services) { + _dbService = services.GetService(); + _module = module; + } + + #region IDependencyProvider + public ISet GetDependencies(PythonAst ast) { + if (_dbService != null && _dbService.TryRestoreDependencies(_module, out var dp)) { + return dp.GetDependencies(ast); + } + + // TODO: try and handle LoadFunctionDependencyModules functionality here. + var dw = new DependencyWalker(_module, ast); + return dw.Dependencies; + } + #endregion + + private sealed class EmptyDependencyProvider: IDependencyProvider { + public ISet GetDependencies(PythonAst ast) => new HashSet(); + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index aeeaa41af..b55efdedf 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations.Typing; @@ -42,7 +43,7 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer { + internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer, ILocationConverter { private enum State { None, Loading, @@ -79,17 +80,19 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser Log = services.GetService(); Interpreter = services.GetService(); Analysis = new EmptyAnalysis(services, this); + GlobalScope = Analysis.GlobalScope; _diagnosticsService = services.GetService(); SetDeclaringModule(this); } - protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, IServiceContainer services) : + protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isPersistent, IServiceContainer services) : this(new ModuleCreationOptions { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType, - Stub = stub + Stub = stub, + IsPersistent = isPersistent }, services) { } internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services) @@ -102,8 +105,10 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s if (uri == null && !string.IsNullOrEmpty(creationOptions.FilePath)) { Uri.TryCreate(creationOptions.FilePath, UriKind.Absolute, out uri); } + Uri = uri; FilePath = creationOptions.FilePath ?? uri?.LocalPath; + Stub = creationOptions.Stub; if (Stub is PythonModule stub && ModuleType != ModuleType.Stub) { stub.PrimaryModule = this; @@ -112,11 +117,14 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s if (ModuleType == ModuleType.Specialized || ModuleType == ModuleType.Unresolved) { ContentState = State.Analyzed; } + + IsPersistent = creationOptions.IsPersistent; InitializeContent(creationOptions.Content, 0); } #region IPythonType public string Name { get; } + public string QualifiedName => ModuleType == ModuleType.Stub ? $"{Name}(stub)" : Name; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; @@ -149,10 +157,10 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => Analysis.GlobalScope.Variables[name]?.Value; + public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; public virtual IEnumerable GetMemberNames() { // drop imported modules and typing. - return Analysis.GlobalScope.Variables + return GlobalScope.Variables .Where(v => { // Instances are always fine. if (v.Value is IPythonInstance) { @@ -162,25 +170,33 @@ public virtual IEnumerable GetMemberNames() { if (valueType is PythonModule) { return false; // Do not re-export modules. } - // Do not re-export types from typing - return !(valueType?.DeclaringModule is TypingModule) || this is TypingModule; + if (valueType is IPythonFunctionType f && f.IsLambda()) { + return false; + } + if (this is TypingModule) { + return true; // Let typing module behave normally. + } + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + if (valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) { + return false; + } + return true; }) .Select(v => v.Name); } #endregion #region ILocatedMember - public override LocationInfo Definition => new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0); - #endregion - - #region IPythonFile - public virtual string FilePath { get; } - public virtual Uri Uri { get; } + public override LocationInfo Definition => Uri != null ? new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0) : LocationInfo.Empty; #endregion #region IPythonModule + public virtual string FilePath { get; protected set; } + public virtual Uri Uri { get; } public IDocumentAnalysis Analysis { get; private set; } - public IPythonInterpreter Interpreter { get; } /// @@ -192,7 +208,7 @@ public virtual IEnumerable GetMemberNames() { /// /// Global cope of the module. /// - public IGlobalScope GlobalScope { get; private set; } + public IGlobalScope GlobalScope { get; protected set; } /// /// If module is a stub points to the primary module. @@ -200,6 +216,11 @@ public virtual IEnumerable GetMemberNames() { /// wants to see library code and not a stub. /// public IPythonModule PrimaryModule { get; private set; } + + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; } #endregion #region IDisposable @@ -210,6 +231,8 @@ protected virtual void Dispose(bool disposing) { _disposeToken.TryMarkDisposed(); var analyzer = Services.GetService(); analyzer.RemoveAnalysis(this); + _parseCts?.Dispose(); + _linkedParseCts?.Dispose(); } #endregion @@ -241,7 +264,7 @@ public string Content { return _buffer.Text; } } - } + } #endregion #region Parsing @@ -381,6 +404,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #endregion #region IAnalyzable + public virtual IDependencyProvider DependencyProvider => new DependencyProvider(this, Services); public void NotifyAnalysisBegins() { lock (_syncObj) { @@ -504,7 +528,11 @@ private void InitializeContent(string content, int version) { private void LoadContent(string content, int version) { if (ContentState < State.Loading) { try { - content = content ?? LoadContent(); + if (IsPersistent) { + content = string.Empty; + } else { + content = content ?? LoadContent(); + } _buffer.Reset(version, content); ContentState = State.Loaded; } catch (IOException) { } catch (UnauthorizedAccessException) { } @@ -562,6 +590,11 @@ private string TryGetDocFromModuleInitFile() { } #endregion + #region ILocationConverter + public virtual SourceLocation IndexToLocation(int index) => this.GetAst()?.IndexToLocation(index) ?? default; + public virtual int LocationToIndex(SourceLocation location) => this.GetAst()?.LocationToIndex(location) ?? default; + #endregion + private void RemoveReferencesInModule(IPythonModule module) { if (module.GlobalScope?.Variables != null) { foreach (var v in module.GlobalScope.Variables) { diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index 0fd55a169..6d4367218 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -16,11 +16,12 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Modules { /// @@ -28,10 +29,12 @@ namespace Microsoft.Python.Analysis.Modules { /// Contains either module members, members + imported children of explicit package or imported implicit package children /// Instance is unique for each module analysis /// - internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEquatable { + internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEquatable, ILocationConverter { private readonly Dictionary _children = new Dictionary(); - + public string Name { get; } + public string QualifiedName => Name; + public IPythonModule Module { get; } public IPythonInterpreter Interpreter { get; } @@ -43,19 +46,20 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public bool IsSpecialized => Module?.IsSpecialized ?? false; public ModuleType ModuleType => Module?.ModuleType ?? ModuleType.Package; public IPythonModule PrimaryModule => null; - public IPythonModule Stub => null; + public IPythonModule Stub => Module?.Stub; public IGlobalScope GlobalScope => Module?.GlobalScope; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public Uri Uri => Module?.Uri; public override PythonMemberType MemberType => PythonMemberType.Module; + public bool IsPersistent => Module?.IsPersistent == true; - public PythonVariableModule(string name, IPythonInterpreter interpreter): base(null) { + public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) { Name = name; Interpreter = interpreter; SetDeclaringModule(this); } - public PythonVariableModule(IPythonModule module): base(module) { + public PythonVariableModule(IPythonModule module): base(module) { Name = module.Name; Interpreter = module.Interpreter; Module = module; @@ -71,5 +75,14 @@ public PythonVariableModule(IPythonModule module): base(module) { public IMember CreateInstance(string typeName = null, IArgumentSet args = null) => this; public bool Equals(IPythonModule other) => other is PythonVariableModule module && Name.EqualsOrdinal(module.Name); + + #region ILocationConverter + public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default; + public int LocationToIndex(SourceLocation location) => (Module as ILocationConverter)?.LocationToIndex(location) ?? default; + #endregion + + #region IDependencyProvider + public IDependencyProvider DependencyProvider => (Module as IAnalyzable)?.DependencyProvider; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 73678cf2c..26c0d3f83 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); private readonly IUIService _ui; + private IModuleDatabaseService _dbService; private IRunningDocumentTable _rdt; private ImmutableArray _userConfiguredPaths; @@ -50,6 +51,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr } public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion); + public ImmutableArray LibraryPaths { get; private set; } = ImmutableArray.Empty; public IBuiltinsPythonModule BuiltinsModule { get; private set; } @@ -68,26 +70,32 @@ protected override IPythonModule CreateModule(string name) { } } - // If there is a stub, make sure it is loaded and attached - // First check stub next to the module. - if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - // If nothing found, try Typeshed. - stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); - } + var dbs = GetDbService(); + moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath); + + IPythonModule stub = null; + if (!moduleImport.IsPersistent) { + // If there is a stub, make sure it is loaded and attached + // First check stub next to the module. + if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { + // If nothing found, try Typeshed. + stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); + } - // If stub is created and its path equals to module, return that stub as module - if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { - return stub; + // If stub is created and its path equals to module, return that stub as module + if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { + return stub; + } } if (moduleImport.IsBuiltin) { Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - return new CompiledBuiltinPythonModule(name, stub, Services); + return new CompiledBuiltinPythonModule(name, stub, moduleImport.IsPersistent, Services); } if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, Services); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, Services); } Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); @@ -97,7 +105,8 @@ protected override IPythonModule CreateModule(string name) { ModuleName = moduleImport.FullName, ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, FilePath = moduleImport.ModulePath, - Stub = stub + Stub = stub, + IsPersistent = moduleImport.IsPersistent }; return GetRdt().AddModule(mco); @@ -130,19 +139,30 @@ private async Task> GetInterpreterSearchPathsA /// /// Module to specialize. /// Specialized module constructor. - /// Original (library) module loaded as stub. - public IPythonModule SpecializeModule(string name, Func specializationConstructor) { + /// Replace existing loaded module, if any. + /// Specialized module. + public IPythonModule SpecializeModule(string name, Func specializationConstructor, bool replaceExisting = false) { var import = CurrentPathResolver.GetModuleImportFromModuleName(name); var module = specializationConstructor(import?.ModulePath); _specialized[name] = module; + + if (replaceExisting) { + Modules.TryRemove(name, out _); + } return module; } /// /// Returns specialized module, if any. /// - public IPythonModule GetSpecializedModule(string name) - => _specialized.TryGetValue(name, out var module) ? module : null; + public IPythonModule GetSpecializedModule(string fullName, bool allowCreation = false, string modulePath = null) + => _specialized.TryGetValue(fullName, out var module) ? module : null; + + /// + /// Determines of module is specialized or exists in the database. + /// + public bool IsSpecializedModule(string fullName, string modulePath = null) + => _specialized.ContainsKey(fullName); internal async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { var analyzer = Services.GetService(); @@ -197,8 +217,8 @@ private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer serv } private async Task ReloadSearchPaths(CancellationToken cancellationToken = default) { - var paths = await GetInterpreterSearchPathsAsync(cancellationToken); - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, paths, _userConfiguredPaths); + LibraryPaths = await GetInterpreterSearchPathsAsync(cancellationToken); + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, LibraryPaths, _userConfiguredPaths); InterpreterPaths = interpreterPaths.Select(p => p.Path); UserPaths = userPaths.Select(p => p.Path); @@ -250,5 +270,8 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu private IRunningDocumentTable GetRdt() => _rdt ?? (_rdt = Services.GetService()); + + private IModuleDatabaseService GetDbService() + => _dbService ?? (_dbService = Services.GetService()); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index e0b2d10d6..e4f8327f8 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -66,15 +66,24 @@ public IPythonModule GetImportedModule(string name) => Modules.TryGetValue(name, out var moduleRef) ? moduleRef.Value : Interpreter.ModuleResolution.GetSpecializedModule(name); public IPythonModule GetOrLoadModule(string name) { - if (Modules.TryGetValue(name, out var moduleRef)) { - return moduleRef.GetOrCreate(name, this); + // Specialized should always win. However, we don't want + // to allow loading from the database just yet since module + // may already exist in the analyzed state. + var module = GetImportedModule(name); + if (module != null) { + return module; } - var module = Interpreter.ModuleResolution.GetSpecializedModule(name); + module = Interpreter.ModuleResolution.GetSpecializedModule(name); if (module != null) { return module; } + // Now try regular case. + if (Modules.TryGetValue(name, out var moduleRef)) { + return moduleRef.GetOrCreate(name, this); + } + moduleRef = Modules.GetOrAdd(name, new ModuleRef()); return moduleRef.GetOrCreate(name, this); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 64762559e..af3ad1884 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -22,7 +21,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index 0e3e5fbcb..771acd25f 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; @@ -31,11 +32,15 @@ namespace Microsoft.Python.Analysis.Modules { /// internal abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) - : base(name, modulePath, ModuleType.Specialized, null, services) { } + : base(name, modulePath, ModuleType.Specialized, null, false, services) { } protected override string LoadContent() { // Exceptions are handled in the base return FileSystem.FileExists(FilePath) ? FileSystem.ReadTextWithRetry(FilePath) : string.Empty; } + + #region IAnalyzable + public override IDependencyProvider DependencyProvider => Modules.DependencyProvider.Empty; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs index 83fa0ffbe..862daf817 100644 --- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs @@ -25,7 +25,7 @@ internal class StubPythonModule : CompiledPythonModule { public bool IsTypeshed { get; } public StubPythonModule(string moduleName, string stubPath, bool isTypeshed, IServiceContainer services) - : base(moduleName, ModuleType.Stub, stubPath, null, services) { + : base(moduleName, ModuleType.Stub, stubPath, null, false, services) { IsTypeshed = isTypeshed; } diff --git a/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs b/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs index 52c4e3665..9aa458a2d 100644 --- a/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs +++ b/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs @@ -15,5 +15,7 @@ using System.Runtime.CompilerServices; +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index 64b8281d9..ea7694ec6 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -20,20 +20,22 @@ using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations { public static class BuiltinsSpecializations { - public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 ? args.FirstOrDefault(a => !a.IsUnknown()) ?? args[0] : null; } - public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); - return args.Count > 0 ? args[0].GetPythonType() : module.Interpreter.GetBuiltinType(BuiltinTypeId.Type); + var t = args.Count > 0 ? args[0].GetPythonType() : module.Interpreter.GetBuiltinType(BuiltinTypeId.Type); + return t.ToBound(); } - public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); if (args.Count > 0) { if (args[0] is IPythonCollection seq) { @@ -47,21 +49,21 @@ public static IMember Iterator(IPythonModule module, IPythonFunctionOverload ove return null; } - public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet) - => PythonCollectionType.CreateList(interpreter, argSet); + public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) + => PythonCollectionType.CreateList(interpreter.ModuleResolution.BuiltinsModule, argSet); - public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var type = new TypingListType("List", module.Interpreter.GetBuiltinType(BuiltinTypeId.Str), module.Interpreter, false); return new TypingList(type); } - public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var str = module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var obj = module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); var type = new TypingDictionaryType("Dict", str, obj, module.Interpreter, false); return new TypingDictionary(type); } - public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 && args[0] is IPythonIterator it ? it.Next : null; } @@ -69,27 +71,27 @@ public static IMember Next(IPythonModule module, IPythonFunctionOverload overloa public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId contentTypeId) { var location = new Location(interpreter.ModuleResolution.BuiltinsModule); var fn = new PythonFunctionType(@"__iter__", location, null, string.Empty); - var o = new PythonFunctionOverload(fn.Name, location); + var o = new PythonFunctionOverload(fn, location); o.AddReturnValue(PythonTypeIterator.FromTypeId(interpreter, contentTypeId)); fn.AddOverload(o); return fn; } - public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); if (args.Count > 0) { - var type = new PythonCollectionType(null, BuiltinTypeId.List, module.Interpreter, false); + var type = new PythonCollectionType(BuiltinTypeId.List, module.Interpreter.ModuleResolution.BuiltinsModule, false); return new PythonCollection(type, new[] { args[0] }); } return null; } - public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var args = argSet.Values(); return args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null; } - public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { var mode = argSet.GetArgumentValue("mode"); var binary = false; @@ -117,7 +119,7 @@ public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverloa return returnType != null ? new PythonInstance(returnType) : null; } - public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet) { + public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) { // TODO: Try __getattr__ first; this may not be as reliable in practice // given we could be assuming that __getattr__ always returns the same type, // which is incorrect more often than not. diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs index e71222bc4..cc58646b3 100644 --- a/src/Analysis/Ast/Impl/Specializations/Specialized.cs +++ b/src/Analysis/Ast/Impl/Specializations/Specialized.cs @@ -19,8 +19,8 @@ namespace Microsoft.Python.Analysis.Specializations { internal static class Specialized { public static IPythonPropertyType Property(string name, IPythonModule declaringModule, IPythonType declaringType, IMember returnValue) { var location = new Location(declaringModule); - var prop = new PythonPropertyType(name, location, declaringType, false); - var o = new PythonFunctionOverload(prop.Name, location); + var prop = new PythonPropertyType(name, location, string.Empty, declaringType, false); + var o = new PythonFunctionOverload(prop, location); o.AddReturnValue(returnValue); prop.AddOverload(o); return prop; @@ -29,7 +29,7 @@ public static IPythonPropertyType Property(string name, IPythonModule declaringM public static IPythonFunctionType Function(string name, IPythonModule declaringModule, string documentation, IMember returnValue) { var location = new Location(declaringModule); var prop = PythonFunctionType.Specialize(name, declaringModule, documentation); - var o = new PythonFunctionOverload(prop.Name, location); + var o = new PythonFunctionOverload(prop, location); o.AddReturnValue(returnValue); prop.AddOverload(o); return prop; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs similarity index 95% rename from src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs rename to src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs index 10cbc03f9..e5ca22b09 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBase.cs @@ -20,7 +20,7 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// /// Represents Generic[T1, T2, ...]. Used as a base class to generic classes. /// - public interface IGenericClassParameter: IPythonType { + public interface IGenericClassBase: IPythonType { /// /// List of T1, T2, ... generic type parameters /// diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs index 1376a6ed1..1b2283de2 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs @@ -32,5 +32,8 @@ public interface IGenericTypeParameter : IPythonType, IEquatable IPythonType Bound { get; } + + object Covariant { get; } + object Contravariant { get; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs index 17d5acbe8..bc22e2a3a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs @@ -20,7 +20,6 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// Represents typing.NamedTuple. /// public interface ITypingNamedTupleType : ITypingTupleType { - string TupleName { get; } IReadOnlyList ItemNames { get; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs index d03d039a2..73eef02f4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/AnyType.cs @@ -24,6 +24,7 @@ public AnyType(IPythonModule declaringModule) : base(declaringModule) { } public override PythonMemberType MemberType => PythonMemberType.Class; public string Name => "Any"; + public string QualifiedName => "typing:Any"; public BuiltinTypeId TypeId => BuiltinTypeId.Type; public string Documentation => Name; public bool IsBuiltin => false; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs similarity index 55% rename from src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs rename to src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs index 164d021d8..7bea95f73 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBase.cs @@ -17,28 +17,32 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Disposables; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { /// - /// Represents Generic[T1, T2, ...] parameter. When class is instantiated + /// Represents Generic[T1, T2, ...] base. When class is instantiated /// or methods evaluated, class generic parameters are matched to /// generic type parameters from TypeVar. /// - internal sealed class GenericClassParameter : PythonClassType, IGenericClassParameter { - internal GenericClassParameter(IReadOnlyList typeArgs, IPythonModule declaringModule) - : base("Generic", new Location(declaringModule)) { - TypeParameters = typeArgs ?? new List(); + internal sealed class GenericClassBase : PythonClassType, IGenericClassBase { + internal GenericClassBase(IReadOnlyList typeArgs, IPythonInterpreter interpreter) + : base("Generic", new Location(interpreter.ModuleResolution.GetSpecializedModule("typing"))) { + TypeParameters = typeArgs ?? Array.Empty(); } - public override bool IsGeneric => true; + #region IPythonType + public override PythonMemberType MemberType => PythonMemberType.Generic; + public override string Documentation => Name; + #endregion - public override IReadOnlyDictionary GenericParameters - => TypeParameters.ToDictionary(tp => tp, tp => tp as IPythonType ?? UnknownType) ?? EmptyDictionary.Instance; + #region IPythonClassType + public override bool IsGeneric => true; + public override IReadOnlyDictionary GenericParameters + => TypeParameters.ToDictionary(tp => tp.Name, tp => tp as IPythonType ?? UnknownType); + public override IPythonType CreateSpecificType(IArgumentSet args) + => new GenericClassBase(args.Arguments.Select(a => a.Value).OfType().ToArray(), DeclaringModule.Interpreter); + #endregion public IReadOnlyList TypeParameters { get; } - - public override PythonMemberType MemberType => PythonMemberType.Generic; } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs index f66b03bb8..49c978c48 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs @@ -25,21 +25,23 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { /// /// Base class for generic types and type declarations. /// - internal class SpecializedGenericType : LocatedMember, IGenericType { + internal sealed class SpecializedGenericType : LocatedMember, IGenericType { internal SpecificTypeConstructor SpecificTypeConstructor { get; } /// /// Constructs generic type with generic type parameters. Typically used /// in generic classes such as when handling Generic[_T] base. /// - public SpecializedGenericType(string name, IReadOnlyList parameters, IPythonModule declaringModule) - : this(name, declaringModule) { + public SpecializedGenericType(string name, string qualifiedName, IReadOnlyList parameters, IPythonModule declaringModule) + : this(name, qualifiedName, declaringModule) { Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); } /// /// Constructs generic type with dynamic type constructor. /// Typically used in type specialization scenarios. + /// Type created with this constructor cannot be persisted + /// since it does not have qualified name. /// /// Type name including parameters, such as Iterator[T] /// Constructor of specific types. @@ -47,44 +49,81 @@ public SpecializedGenericType(string name, IReadOnlyList /// Type id. Used in type comparisons such as when matching /// function arguments. For example, Iterator[T] normally has type id of ListIterator. /// Optional type parameters as declared by TypeVar. + /// Optional documentation. Defaults to . public SpecializedGenericType( string name, SpecificTypeConstructor specificTypeConstructor, IPythonModule declaringModule, BuiltinTypeId typeId = BuiltinTypeId.Unknown, - IReadOnlyList parameters = null - ) : this(name, declaringModule) { + IReadOnlyList parameters = null, + string documentation = null + ) : this(name, null, declaringModule) { SpecificTypeConstructor = specificTypeConstructor ?? throw new ArgumentNullException(nameof(specificTypeConstructor)); TypeId = typeId; Parameters = parameters ?? Array.Empty(); + Documentation = documentation ?? name; } - private SpecializedGenericType(string name, IPythonModule declaringModule) : base(declaringModule) { + /// + /// Constructs generic type with dynamic type constructor. + /// Typically used in type specialization scenarios. + /// + /// Type name including parameters, such as Iterator[T] + /// Qualified type name including parameters, such as typing:Iterator[module:T] + /// Constructor of specific types. + /// Declaring module. + /// Type id. Used in type comparisons such as when matching + /// function arguments. For example, Iterator[T] normally has type id of ListIterator. + /// Optional type parameters as declared by TypeVar. + /// Optional documentation. Defaults to . + public SpecializedGenericType( + string name, + string qualifiedName, + SpecificTypeConstructor specificTypeConstructor, + IPythonModule declaringModule, + BuiltinTypeId typeId = BuiltinTypeId.Unknown, + IReadOnlyList parameters = null, + string documentation = null + ) : this(name, qualifiedName, declaringModule) { + SpecificTypeConstructor = specificTypeConstructor ?? throw new ArgumentNullException(nameof(specificTypeConstructor)); + TypeId = typeId; + Parameters = parameters ?? Array.Empty(); + Documentation = documentation ?? name; + } + + private SpecializedGenericType(string name, string qualifiedName, IPythonModule declaringModule) + : base(declaringModule) { Name = name ?? throw new ArgumentNullException(nameof(name)); + QualifiedName = qualifiedName ?? $"{declaringModule.Name}:{name}"; + Documentation = Name; } + #region IMember public override PythonMemberType MemberType => PythonMemberType.Generic; + #endregion + #region IGenericType /// /// Type parameters such as in Tuple[T1, T2. ...] or /// Generic[_T1, _T2, ...] as returned by TypeVar. /// public IReadOnlyList Parameters { get; } + public bool IsGeneric => true; + #endregion #region IPythonType public string Name { get; } + public string QualifiedName { get; } public IMember GetMember(string name) => null; public IEnumerable GetMemberNames() => Enumerable.Empty(); public BuiltinTypeId TypeId { get; } = BuiltinTypeId.Unknown; - public virtual string Documentation => Name; + public string Documentation { get; } public bool IsBuiltin => false; public bool IsAbstract => true; public bool IsSpecialized => true; - public bool IsGeneric => true; - public IMember CreateInstance(string typeName, IArgumentSet args) { - var types = args.Values(); + var types = GetTypesFromValues(args.Arguments); if (types.Count != args.Arguments.Count) { throw new ArgumentException(@"Generic type instance construction arguments must be all of IPythonType", nameof(args)); } @@ -94,8 +133,8 @@ public IMember CreateInstance(string typeName, IArgumentSet args) { : specific.CreateInstance(typeName); } - public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; - public virtual IMember Index(IPythonInstance instance, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; + public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; + public IMember Index(IPythonInstance instance, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; /// /// Creates instance of a type information with the specific @@ -117,5 +156,8 @@ public override bool Equals(object other) { public override int GetHashCode() => TypeId != BuiltinTypeId.Unknown ? TypeId.GetHashCode() : base.GetHashCode(); + + private IReadOnlyList GetTypesFromValues(IEnumerable args) + => args.Select(a => a.Value).OfType().Select(m => m.GetPythonType()).ToArray(); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index eb7a59412..d59dd27ac 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -25,22 +25,37 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class GenericTypeParameter : PythonType, IGenericTypeParameter { - public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnlyList constraints, - IPythonType bound, string documentation, IndexSpan location) - : base(name, new Location(declaringModule), documentation) { + public GenericTypeParameter( + string name, + IPythonModule declaringModule, + IReadOnlyList constraints, + IPythonType bound, + object covariant, + object contravariant, + IndexSpan indexSpan) + : base(name, new Location(declaringModule, indexSpan), + GetDocumentation(name, constraints, bound, covariant, contravariant, declaringModule)) { Constraints = constraints ?? Array.Empty(); Bound = bound; + Covariant = covariant; + Contravariant = contravariant; } #region IGenericTypeParameter public IReadOnlyList Constraints { get; } public IPythonType Bound { get; } + public object Covariant { get; } + public object Contravariant { get; } #endregion + #region IPythonType + public override BuiltinTypeId TypeId => BuiltinTypeId.Type; public override PythonMemberType MemberType => PythonMemberType.Generic; public override bool IsSpecialized => true; + #endregion + private static bool TypeVarArgumentsValid(IArgumentSet argSet) { var args = argSet.Arguments; var constraints = argSet.ListArgument?.Values ?? Array.Empty(); @@ -92,19 +107,22 @@ private static IPythonType GetBoundType(IArgumentSet argSet) { switch (rawBound) { case IPythonType t: return t; - case IPythonConstant c when c.GetString() != null: - return eval.GetTypeFromString(c.GetString()); + case IPythonConstant c: + var s = c.GetString(); + if (!string.IsNullOrEmpty(s)) { + return eval.GetTypeFromString(s) ?? argSet.Eval.UnknownType; + } + return argSet.Eval.UnknownType; default: return rawBound.GetPythonType(); } } - public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, IndexSpan location = default) { + public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, IndexSpan indexSpan = default) { if (!TypeVarArgumentsValid(argSet)) { return declaringModule.Interpreter.UnknownType; } - var args = argSet.Arguments; var constraintArgs = argSet.ListArgument?.Values ?? Array.Empty(); var name = argSet.GetArgumentValue("name")?.GetString(); @@ -112,24 +130,39 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari // Type constraints may be specified as type name strings. var typeString = a.GetString(); return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType(); - }).ToArray() ?? Array.Empty(); + }).ToArray(); + var bound = GetBoundType(argSet); - var documentation = GetDocumentation(args, constraints); + var covariant = argSet.GetArgumentValue("covariant")?.Value; + var contravariant = argSet.GetArgumentValue("contravariant")?.Value; - return new GenericTypeParameter(name, declaringModule, constraints, bound, documentation, location); + return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, indexSpan); } - private static string GetDocumentation(IReadOnlyList args, IReadOnlyList constraints) { - var name = (args[0].Value as IPythonConstant).GetString(); - var keyWordArgs = args.Skip(1) - .Where(x => !x.ValueIsDefault) - .Select(x => $"{x.Name}={(x.Value as IPythonConstant)?.Value}"); + private static string GetDocumentation(string name, IReadOnlyList constraints, IPythonType bound, object covariant, object contravariant, IPythonModule declaringModule) { + var constaintStrings = constraints != null ? constraints.Select(c => c.IsUnknown() ? "?" : c.Name) : Enumerable.Empty(); + + var boundStrings = Enumerable.Empty(); + if (bound != null) { + string boundName; + if(bound.DeclaringModule.Equals(declaringModule) || bound.DeclaringModule is IBuiltinsPythonModule) { + boundName = bound.Name; + } else { + boundName = $"{bound.DeclaringModule.Name}.{bound.Name}"; + } + boundStrings = Enumerable.Repeat($"bound={boundName}", 1); + } + + var covariantStrings = covariant != null ? Enumerable.Repeat($"covariant={covariant}", 1) : Enumerable.Empty(); + var contravariantStrings = contravariant != null ? Enumerable.Repeat($"contravariant={contravariant}", 1) : Enumerable.Empty(); + + var docArgs = Enumerable.Repeat($"'{name}'", 1) + .Concat(constaintStrings).Concat(boundStrings).Concat(covariantStrings).Concat(contravariantStrings); - var docArgs = constraints.Select(c => c.IsUnknown() ? "?" : c.Name).Concat(keyWordArgs).Prepend($"'{name}'"); var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs); return documentation; } - public bool Equals(IGenericTypeParameter other) => Name.Equals(other.Name); + public bool Equals(IGenericTypeParameter other) => Name.Equals(other?.Name); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs index 2343bec6b..2ecd670e5 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs @@ -21,27 +21,50 @@ using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class NamedTupleType : TypingTupleType, ITypingNamedTupleType { + // Since named tuple operates as a new, separate type, we need to track + // its location rather than delegating down to the general wrapper over + // Python built-in tuple. + private sealed class NamedTupleLocatedMember: LocatedMember { + public NamedTupleLocatedMember(Location location) : base(location) { } + public override PythonMemberType MemberType => PythonMemberType.Class; + } + private readonly NamedTupleLocatedMember _locatedMember; + /// /// Creates type info for a strongly-typed tuple, such as Tuple[T1, T2, ...]. /// - public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonInterpreter interpreter) - : base(itemTypes, interpreter) { - TupleName = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); + public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IndexSpan indexSpan) + : base(itemTypes, declaringModule, declaringModule.Interpreter) { + Name = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); ItemNames = itemNames; var typeNames = itemTypes.Select(t => t.IsUnknown() ? string.Empty : t.Name); var pairs = itemNames.Zip(typeNames, (name, typeName) => string.IsNullOrEmpty(typeName) ? name : $"{name}: {typeName}"); - Name = CodeFormatter.FormatSequence(tupleName, '(', pairs); + Documentation = CodeFormatter.FormatSequence(tupleName, '(', pairs); + + _locatedMember = new NamedTupleLocatedMember(new Location(declaringModule, indexSpan)); } - public string TupleName { get; } public IReadOnlyList ItemNames { get; } + #region IPythonType public override string Name { get; } + public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; // Named tuple name is a type name as class. public override bool IsSpecialized => true; + public override string Documentation { get; } + #endregion + + #region ILocatedMember + public override Location Location => _locatedMember.Location; + public override LocationInfo Definition => _locatedMember.Definition; + public override IReadOnlyList References => _locatedMember.References; + public override void AddReference(Location location) => _locatedMember.AddReference(location); + public override void RemoveReferences(IPythonModule module) => _locatedMember.RemoveReferences(module); + #endregion public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingTuple(this); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs index a4ba593a8..0c89c3a7a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/OptionalType.cs @@ -22,8 +22,10 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class OptionalType : PythonTypeWrapper, IPythonUnionType { public OptionalType(IPythonModule declaringModule, IPythonType type) : base(type, declaringModule) { Name = $"Optional[{type.Name}]"; + QualifiedName = $"typing:Optional[{type.QualifiedName}]"; } public override string Name { get; } + public override string QualifiedName { get; } public override PythonMemberType MemberType => PythonMemberType.Union; public override bool IsSpecialized => true; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypeAlias.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypeAlias.cs index ffcbde6a1..67d93be1c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypeAlias.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypeAlias.cs @@ -21,6 +21,7 @@ public TypeAlias(string name, IPythonType type) : base(type) { Name = name; } public override string Name { get; } + public override string QualifiedName => $"typing:{Name}"; public override bool IsSpecialized => true; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs index 360c7a6f5..6f0e55bd4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingDictionaryType.cs @@ -31,13 +31,14 @@ internal class TypingDictionaryType : PythonDictionaryType, ITypingDictionaryTyp /// Type name (Dict, Mapping, ...) /// Type of dictionary keys. /// Type of dictionary values. - /// Python interpreter + /// Python interpreter. /// Tells if collection is mutable (Dict) or not (Mapping) public TypingDictionaryType(string name, IPythonType keyType, IPythonType valueType, IPythonInterpreter interpreter, bool isMutable) - : base(interpreter, isMutable) { + : base(interpreter.ModuleResolution.GetSpecializedModule("typing"), isMutable) { KeyType = keyType; ValueType = valueType; Name = $"{name}[{keyType.Name}, {valueType.Name}]"; + QualifiedName = $"typing:{name}[{keyType.QualifiedName}, {valueType.QualifiedName}]"; } public IPythonType KeyType { get; } @@ -45,13 +46,14 @@ public TypingDictionaryType(string name, IPythonType keyType, IPythonType valueT public IPythonType ItemType => _itemType ?? (_itemType = CreateItemType()); public override string Name { get; } + public override string QualifiedName { get; } public override IMember CreateInstance(string typeName, IArgumentSet args) => new TypingDictionary(this); public override IMember Index(IPythonInstance instance, IArgumentSet args) => new PythonInstance(ValueType); public override bool IsSpecialized => true; private TypingTupleType CreateItemType() { - var itemType = new TypingTupleType(new[] { KeyType, ValueType }, DeclaringModule.Interpreter); + var itemType = new TypingTupleType(new[] { KeyType, ValueType }, DeclaringModule, DeclaringModule.Interpreter); return itemType; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs index 81762ac42..1b4fa245f 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs @@ -30,10 +30,11 @@ internal sealed class TypingIteratorType : PythonIteratorType, ITypingIteratorTy /// track items in the collection, it repeats the same item endlessly. /// public TypingIteratorType(IPythonType itemType, BuiltinTypeId iteratorType, IPythonInterpreter interpreter) - : base(iteratorType, interpreter) { + : base(iteratorType, interpreter.ModuleResolution.GetSpecializedModule("typing")) { ItemTypes = new[] { itemType }; Repeat = true; Name = $"Iterator[{itemType.Name}]"; + QualifiedName = $"typing:Iterator[{itemType.QualifiedName}]"; } /// @@ -41,14 +42,16 @@ public TypingIteratorType(IPythonType itemType, BuiltinTypeId iteratorType, IPyt /// The iterator goes along declared items types and stops when there are no more types. /// public TypingIteratorType(IReadOnlyList itemTypes, BuiltinTypeId iteratorType, IPythonInterpreter interpreter) - : base(iteratorType, interpreter) { + : base(iteratorType, interpreter.ModuleResolution.GetSpecializedModule("typing")) { ItemTypes = itemTypes; Name = $"Iterator[{CodeFormatter.FormatSequence(string.Empty, '(', itemTypes)}]"; + QualifiedName = $"typing:Iterator[{CodeFormatter.FormatSequence(string.Empty, '(', itemTypes.Select(t => t.QualifiedName))}]"; } public IReadOnlyList ItemTypes { get; } public bool Repeat { get; } public override string Name { get; } + public override string QualifiedName { get; } public override bool IsSpecialized => true; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs index e627c6338..2c1d3ba27 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingListType.cs @@ -25,7 +25,7 @@ internal class TypingListType : PythonCollectionType, ITypingListType { /// /// Type name. /// List item type. - /// Python interpreter + /// Python interpreter. /// Tells of list represents a mutable collection. /// If true, type will append item type names to the base type name. public TypingListType(string typeName, IPythonType itemType, IPythonInterpreter interpreter, bool isMutable, bool formatName = true) @@ -37,16 +37,18 @@ public TypingListType(string typeName, IPythonType itemType, IPythonInterpreter /// Type name. /// Collection type id. Can be used when list is used to simulate other collections, like a set. /// List item type. - /// Python interpreter + /// Python interpreter. /// Tells of list represents a mutable collection. /// If true, type will append item type names to the base type name. public TypingListType(string typeName, BuiltinTypeId typeId, IPythonType itemType, IPythonInterpreter interpreter, bool isMutable, bool formatName = true) - : base(null, typeId, interpreter, isMutable) { + : base(typeId, interpreter.ModuleResolution.GetSpecializedModule("typing"), isMutable) { ItemType = itemType; Name = formatName ? $"{typeName}[{itemType.Name}]" : typeName; + QualifiedName = formatName ? $"typing:{typeName}[{itemType.QualifiedName}]" : $"typing:{typeName}"; } public override string Name { get; } + public override string QualifiedName { get; } public override bool IsAbstract => false; public override bool IsSpecialized => true; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs index 838569e7f..52358c16c 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingTupleType.cs @@ -28,16 +28,20 @@ internal class TypingTupleType : PythonCollectionType, ITypingTupleType { /// Creates type info for a strongly-typed tuple, such as Tuple[T1, T2, ...]. /// /// Tuple item types. + /// Declaring module. If null, then 'typing' is assumed. /// Python interpreter. - public TypingTupleType(IReadOnlyList itemTypes, IPythonInterpreter interpreter) - : base(null, BuiltinTypeId.Tuple, interpreter, false) { - ItemTypes = itemTypes; - Name = CodeFormatter.FormatSequence("Tuple", '[', itemTypes); + public TypingTupleType(IReadOnlyList itemTypes, IPythonModule declaringModule, IPythonInterpreter interpreter) + : base(BuiltinTypeId.Tuple, declaringModule ?? interpreter.ModuleResolution.GetSpecializedModule("typing"), false) { + ItemTypes = itemTypes.Count > 0 ? itemTypes : new[] { interpreter.UnknownType }; + Name = CodeFormatter.FormatSequence("Tuple", '[', ItemTypes); + QualifiedName = CodeFormatter.FormatSequence("typing:Tuple", '[', ItemTypes.Select(t => t.QualifiedName)); } public IReadOnlyList ItemTypes { get; } public override string Name { get; } + public override string QualifiedName { get; } + public override bool IsAbstract => false; public override bool IsSpecialized => true; @@ -73,7 +77,7 @@ public override bool Equals(object obj) { return true; } - public override int GetHashCode() + public override int GetHashCode() => ItemTypes.Aggregate(0, (current, item) => current ^ item.GetHashCode()) ^ Name.GetHashCode(); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index 124a89306..1761b4a01 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -23,6 +23,7 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -50,7 +51,7 @@ private void SpecializeMembers() { // TypeVar var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar")); - var o = new PythonFunctionOverload(fn.Name, location); + var o = new PythonFunctionOverload(fn, location); o.SetParameters(new List { new ParameterInfo("name", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null), new ParameterInfo("constraints", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.List, null), @@ -61,8 +62,8 @@ private void SpecializeMembers() { // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, args) - => GenericTypeParameter.FromTypeVar(args, interpreter)); + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) + => GenericTypeParameter.FromTypeVar(args, declaringModule, indexSpan)); fn.AddOverload(o); _members["TypeVar"] = fn; @@ -72,10 +73,10 @@ private void SpecializeMembers() { // Type fn = PythonFunctionType.Specialize("Type", this, GetMemberDocumentation("Type")); - o = new PythonFunctionOverload(fn.Name, location); + o = new PythonFunctionOverload(fn, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, args) => { + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) => { var a = args.Values(); return a.Count == 1 ? a[0] : Interpreter.UnknownType; }); @@ -98,6 +99,8 @@ private void SpecializeMembers() { _members["ValuesView"] = new SpecializedGenericType("ValuesView", CreateValuesViewType, this); _members["ItemsView"] = new SpecializedGenericType("ItemsView", CreateItemsViewType, this); + _members["AbstractSet"] = new SpecializedGenericType("AbstractSet", + typeArgs => CreateListType("AbstractSet", BuiltinTypeId.Set, typeArgs, true), this); _members["Set"] = new SpecializedGenericType("Set", typeArgs => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true), this); _members["MutableSet"] = new SpecializedGenericType("MutableSet", @@ -123,26 +126,27 @@ private void SpecializeMembers() { _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); - _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); - _members["SupportsFloat"] = Interpreter.GetBuiltinType(BuiltinTypeId.Float); - _members["SupportsComplex"] = Interpreter.GetBuiltinType(BuiltinTypeId.Complex); - _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); + // TODO: make these classes that support __float__, etc per spec. + //_members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); + //_members["SupportsFloat"] = Interpreter.GetBuiltinType(BuiltinTypeId.Float); + //_members["SupportsComplex"] = Interpreter.GetBuiltinType(BuiltinTypeId.Complex); + //_members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); fn = PythonFunctionType.Specialize("NamedTuple", this, GetMemberDocumentation("NamedTuple")); - o = new PythonFunctionOverload(fn.Name, location); - o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values())); + o = new PythonFunctionOverload(fn, location); + o.SetReturnValueProvider((declaringModule, overload, args, indexSpan) + => CreateNamedTuple(args.Values(), declaringModule, indexSpan)); fn.AddOverload(o); _members["NamedTuple"] = fn; _members["Any"] = new AnyType(this); - _members["AnyStr"] = CreateAnyStr(); _members["Optional"] = new SpecializedGenericType("Optional", CreateOptional, this); _members["Type"] = new SpecializedGenericType("Type", CreateType, this); - _members["Generic"] = new SpecializedGenericType("Generic", CreateGenericClassParameter, this); + _members["Generic"] = new SpecializedGenericType("Generic", CreateGenericClassBase, this); } private string GetMemberDocumentation(string name) @@ -150,10 +154,10 @@ private string GetMemberDocumentation(string name) private IPythonType SpecializeNewType(Location location) { var fn = PythonFunctionType.Specialize("NewType", this, GetMemberDocumentation("NewType")); - var o = new PythonFunctionOverload(fn.Name, location); + var o = new PythonFunctionOverload(fn, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, args) => CreateTypeAlias(args)); + o.SetReturnValueProvider((interpreter, overload, args, indexSpan) => CreateTypeAlias(args)); o.SetParameters(new[] { new ParameterInfo("name", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null), new ParameterInfo("tp", Interpreter.GetBuiltinType(BuiltinTypeId.Type), ParameterKind.Normal, null), @@ -261,7 +265,7 @@ private IPythonType CreateTypeAlias(IArgumentSet argSet) { eval.ReportDiagnostics( eval.Module?.Uri, new DiagnosticsEntry(Resources.NewTypeFirstArgument, - eval?.GetLocation(argExpr).Span ?? default, + eval.GetLocation(argExpr).Span, Diagnostics.ErrorCodes.TypingNewTypeArguments, Severity.Warning, DiagnosticSource.Analysis) ); @@ -278,7 +282,7 @@ private IPythonType CreateUnion(IReadOnlyList typeArgs) { return Interpreter.UnknownType; } - private IPythonType CreateNamedTuple(IReadOnlyList typeArgs) { + private IPythonType CreateNamedTuple(IReadOnlyList typeArgs, IPythonModule declaringModule, IndexSpan indexSpan) { if (typeArgs.Count != 2) { // TODO: report wrong number of arguments return Interpreter.UnknownType; @@ -323,7 +327,7 @@ private IPythonType CreateNamedTuple(IReadOnlyList typeArgs) { itemNames.Add(itemName2); itemTypes.Add(c.Contents[1].GetPythonType()); } - return TypingTypeFactory.CreateNamedTupleType(Interpreter, tupleName, itemNames, itemTypes); + return TypingTypeFactory.CreateNamedTupleType(tupleName, itemNames, itemTypes, declaringModule, indexSpan); } private IPythonType CreateOptional(IReadOnlyList typeArgs) { @@ -346,24 +350,23 @@ private IPythonType CreateAnyStr() { var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str); var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode); - var name = "AnyStr"; var constraints = Interpreter.LanguageVersion.Is3x() - ? new IPythonType[] { str, bytes } - : new IPythonType[] { str, unicode }; - var docArgs = new[] { $"'{name}'" }.Concat(constraints.Select(c => c.Name)); - var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs); + ? new[] { str, bytes } + : new[] { str, unicode }; + var docArgs = new[] { "'AnyStr'" }.Concat(constraints.Select(c => c.Name)); - return new GenericTypeParameter(name, this, constraints, default, documentation, default); + var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs); + return new PythonTypeWrapper("AnyStr", documentation, this, Interpreter.GetBuiltinType(BuiltinTypeId.Str)); } - private IPythonType CreateGenericClassParameter(IReadOnlyList typeArgs) { + private IPythonType CreateGenericClassBase(IReadOnlyList typeArgs) { // Handle Generic[_T1, _T2, ...]. _T1, et al are IGenericTypeParameter from TypeVar. // Hold the parameter until concrete type is provided at the time of the class instantiation. if (typeArgs.Count > 0) { var typeDefs = typeArgs.OfType().ToArray(); if (typeDefs.Length == typeArgs.Count) { - return new GenericClassParameter(typeDefs, this); + return new GenericClassBase(typeDefs, Interpreter); } else { // TODO: report argument mismatch } @@ -372,10 +375,14 @@ private IPythonType CreateGenericClassParameter(IReadOnlyList typeA return Interpreter.UnknownType; } - private IPythonType ToGenericTemplate(string typeName, IReadOnlyList typeArgs, BuiltinTypeId typeId) - => _members[typeName] is SpecializedGenericType gt - ? new SpecializedGenericType(CodeFormatter.FormatSequence(typeName, '[', typeArgs), gt.SpecificTypeConstructor, this, typeId, - typeArgs.OfType().ToList()) - : Interpreter.UnknownType; + private IPythonType ToGenericTemplate(string typeName, IReadOnlyList typeArgs, BuiltinTypeId typeId) { + if (_members[typeName] is SpecializedGenericType gt) { + var name = CodeFormatter.FormatSequence(typeName, '[', typeArgs); + var qualifiedName = CodeFormatter.FormatSequence($"typing:{typeName}", '[', typeArgs.Select(t => t.QualifiedName)); + var args = typeArgs.OfType().ToList(); + return new SpecializedGenericType(name, qualifiedName, gt.SpecificTypeConstructor, this, typeId, args); + } + return Interpreter.UnknownType; + } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs index afed8ded6..d78037e8a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingTypeFactory.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Specializations.Typing.Values; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Specializations.Typing { internal static class TypingTypeFactory { @@ -26,7 +27,7 @@ public static ITypingListType CreateListType(IPythonInterpreter interpreter, str => new TypingListType(typeName, typeId, itemType, interpreter, isMutable); public static ITypingTupleType CreateTupleType(IPythonInterpreter interpreter, IReadOnlyList types) - => new TypingTupleType(types, interpreter); + => new TypingTupleType(types, null, interpreter); public static ITypingIteratorType CreateIteratorType(IPythonInterpreter interpreter, IPythonType itemType) => new TypingIteratorType(itemType, BuiltinTypeId.ListIterator, interpreter); @@ -55,11 +56,8 @@ public static ITypingListType CreateItemsViewType(IPythonInterpreter interpreter public static IPythonType CreateUnionType(IPythonInterpreter interpreter, IReadOnlyList types, IPythonModule declaringModule) => new PythonUnionType(types.Select(a => a.GetPythonType()), declaringModule); - public static ITypingNamedTupleType CreateNamedTupleType(IPythonInterpreter interpreter, string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes) - => new NamedTupleType(tupleName, itemNames, itemTypes, interpreter); - - public static IPythonType CreateOptionalType(IPythonModule declaringModule, IPythonType type) - => new OptionalType(declaringModule, type); + public static ITypingNamedTupleType CreateNamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IndexSpan indexSpan) + => new NamedTupleType(tupleName, itemNames, itemTypes, declaringModule, indexSpan); public static IPythonType CreateType(IPythonModule declaringModule, IPythonType type) => new TypingType(declaringModule, type); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs index 73fc834a3..e062dacf2 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingType.cs @@ -28,10 +28,12 @@ internal sealed class TypingType : LocatedMember, IPythonType { public TypingType(IPythonModule declaringModule, IPythonType type): base(declaringModule) { _type = type ?? throw new ArgumentNullException(nameof(type)); Name = $"Type[{_type.Name}]"; + QualifiedName = $"{DeclaringModule.Name}:Type[{_type.QualifiedName}]"; } public override PythonMemberType MemberType => PythonMemberType.Class; public string Name { get; } + public string QualifiedName { get; } public BuiltinTypeId TypeId => BuiltinTypeId.Type; public string Documentation => Name; public bool IsBuiltin => false; diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 13bfb7d39..fa4c1bebf 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -19,7 +19,6 @@ using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing; @@ -63,21 +62,16 @@ public static ArgumentSet Empty(Expression expr, IExpressionEvaluator eval) { } /// - /// Creates a set of arguments for a call - /// - /// Use in the cases a corresponding function is unknown, but it is still convenient to have the context - /// of the expression which the arguments are needed for and the evaluator that is analyzing - /// that expression. - /// + /// Creates a set of arguments for a call. /// /// Arguments for the call. /// Expression for the call. - /// Evaluator of the current analysis. + /// Evaluator of the current analysis of arguments are to be evaluated. + /// Can be null if arguments are already known. public ArgumentSet(IReadOnlyList args, Expression expr, IExpressionEvaluator eval) { _arguments = args.Select(t => new Argument(t)).ToList(); Expression = expr; Eval = eval; - _evaluated = true; } @@ -331,7 +325,7 @@ private IMember GetArgumentValue(Argument arg) { } if (arg.ValueIsDefault) { - using (Eval.OpenScope(DeclaringModule.Analysis.GlobalScope)) { + using (Eval.OpenScope(DeclaringModule.GlobalScope)) { return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType; } } diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs index 42ae754f0..839d7adf2 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonCollectionType.cs @@ -16,33 +16,30 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types.Collections { /// /// Type info for an iterable entity. Most base collection class. /// internal class PythonCollectionType : PythonTypeWrapper, IPythonCollectionType { - private string _typeName; - /// /// Creates type info for an collection. /// - /// Iterable type name. If null, name of the type id will be used. /// Collection type id, such as . - /// Python interpreter. + /// Declaring module. /// Indicates if collection is mutable (like list) or immutable (like tuple). public PythonCollectionType( - string typeName, BuiltinTypeId collectionTypeId, - IPythonInterpreter interpreter, + IPythonModule declaringModule, bool isMutable - ) : base(collectionTypeId, interpreter.ModuleResolution.BuiltinsModule) { - _typeName = typeName; + ) : base(collectionTypeId, declaringModule) { TypeId = collectionTypeId; - IteratorType = new PythonIteratorType(collectionTypeId.GetIteratorTypeId(), interpreter); + IteratorType = new PythonIteratorType(collectionTypeId.GetIteratorTypeId(), declaringModule); IsMutable = isMutable; } @@ -56,18 +53,6 @@ bool isMutable #endregion #region IPythonType - public override string Name { - get { - if (_typeName == null) { - var type = DeclaringModule.Interpreter.GetBuiltinType(TypeId); - if (!type.IsUnknown()) { - _typeName = type.Name; - } - } - return _typeName ?? ""; ; - } - } - public override BuiltinTypeId TypeId { get; } public override PythonMemberType MemberType => PythonMemberType.Class; public override IMember GetMember(string name) => name == @"__iter__" ? IteratorType : base.GetMember(name); @@ -80,10 +65,28 @@ public override IMember Call(IPythonInstance instance, string memberName, IArgum public override IMember Index(IPythonInstance instance, IArgumentSet args) => (instance as IPythonCollection)?.Index(args) ?? UnknownType; + + public IPythonType CreateSpecificType(IArgumentSet typeArguments) { + throw new NotImplementedException(); + } + #endregion + #region IGenericType + public IReadOnlyList Parameters => (InnerType as IGenericType)?.Parameters ?? Array.Empty(); + public bool IsGeneric => (InnerType as IPythonClassType)?.IsGeneric == true; + public IReadOnlyDictionary GenericParameters + => (InnerType as IPythonClassType)?.GenericParameters ?? EmptyDictionary.Instance; + #endregion + + #region IPythonClassType + public IPythonType DeclaringType => (InnerType as IPythonClassType)?.DeclaringType; + public ClassDefinition ClassDefinition => (InnerType as IPythonClassType)?.ClassDefinition; + public IReadOnlyList Mro => (InnerType as IPythonClassType)?.Mro ?? Array.Empty(); + public IReadOnlyList Bases => (InnerType as IPythonClassType)?.Bases ?? Array.Empty(); + #endregion - public static IPythonCollection CreateList(IPythonInterpreter interpreter, IArgumentSet args) { + public static IPythonCollection CreateList(IPythonModule declaringModule, IArgumentSet args) { var exact = true; IReadOnlyList contents; if (args.Arguments.Count > 1) { @@ -94,37 +97,37 @@ public static IPythonCollection CreateList(IPythonInterpreter interpreter, IArgu } else { contents = args.ListArgument?.Values; } - return CreateList(interpreter, contents ?? Array.Empty(), exact: exact); + return CreateList(declaringModule, contents ?? Array.Empty(), exact: exact); } - public static IPythonCollection CreateList(IPythonInterpreter interpreter, IReadOnlyList contents, bool flatten = true, bool exact = false) { - var collectionType = new PythonCollectionType(null, BuiltinTypeId.List, interpreter, true); + public static IPythonCollection CreateList(IPythonModule declaringModule, IReadOnlyList contents, bool flatten = true, bool exact = false) { + var collectionType = new PythonCollectionType(BuiltinTypeId.List, declaringModule, true); return new PythonCollection(collectionType, contents, flatten, exact: exact); } - public static IPythonCollection CreateConcatenatedList(IPythonInterpreter interpreter, params IPythonCollection[] many) { + public static IPythonCollection CreateConcatenatedList(IPythonModule declaringModule, params IPythonCollection[] many) { var exact = many?.All(c => c != null && c.IsExact) ?? false; var contents = many?.ExcludeDefault().Select(c => c.Contents).SelectMany().ToList() ?? new List(); - return CreateList(interpreter, contents, false, exact: exact); + return CreateList(declaringModule, contents, false, exact: exact); } - public static IPythonCollection CreateTuple(IPythonInterpreter interpreter, IReadOnlyList contents, bool exact = false) { - var collectionType = new PythonCollectionType(null, BuiltinTypeId.Tuple, interpreter, false); + public static IPythonCollection CreateTuple(IPythonModule declaringModule, IReadOnlyList contents, bool exact = false) { + var collectionType = new PythonCollectionType(BuiltinTypeId.Tuple, declaringModule, false); return new PythonCollection(collectionType, contents, exact: exact); } - public static IPythonCollection CreateConcatenatedTuple(IPythonInterpreter interpreter, params IPythonCollection[] many) { + public static IPythonCollection CreateConcatenatedTuple(IPythonModule declaringModule, params IPythonCollection[] many) { var exact = many?.All(c => c != null && c.IsExact) ?? false; var contents = many?.ExcludeDefault().Select(c => c.Contents).SelectMany().ToList() ?? new List(); - return CreateTuple(interpreter, contents, exact: exact); + return CreateTuple(declaringModule, contents, exact: exact); } - public static IPythonCollection CreateSet(IPythonInterpreter interpreter, IReadOnlyList contents, bool flatten = true, bool exact = false) { - var collectionType = new PythonCollectionType(null, BuiltinTypeId.Set, interpreter, true); + public static IPythonCollection CreateSet(IPythonModule declaringModule, IReadOnlyList contents, bool flatten = true, bool exact = false) { + var collectionType = new PythonCollectionType(BuiltinTypeId.Set, declaringModule, true); return new PythonCollection(collectionType, contents, flatten, exact: exact); } - public override bool Equals(object obj) + public override bool Equals(object obj) => obj is IPythonType pt && (PythonTypeComparer.Instance.Equals(pt, this) || PythonTypeComparer.Instance.Equals(pt, InnerType)); public override int GetHashCode() => PythonTypeComparer.Instance.GetHashCode(this); } diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs index e012e4b3a..a4835deec 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonDictionaryType.cs @@ -19,8 +19,8 @@ namespace Microsoft.Python.Analysis.Types.Collections { internal class PythonDictionaryType : PythonCollectionType { - public PythonDictionaryType(IPythonInterpreter interpreter, bool isMutable = true) - : base(null, BuiltinTypeId.Dict, interpreter, isMutable) { + public PythonDictionaryType(IPythonModule declaringModule, bool isMutable = true) + : base(BuiltinTypeId.Dict, declaringModule, isMutable) { } public override IMember CreateInstance(string typeName, IArgumentSet args) { diff --git a/src/Analysis/Ast/Impl/Types/Collections/PythonIteratorType.cs b/src/Analysis/Ast/Impl/Types/Collections/PythonIteratorType.cs index 865cebae9..67f5df3b8 100644 --- a/src/Analysis/Ast/Impl/Types/Collections/PythonIteratorType.cs +++ b/src/Analysis/Ast/Impl/Types/Collections/PythonIteratorType.cs @@ -26,9 +26,9 @@ internal class PythonIteratorType : PythonTypeWrapper, IPythonIteratorType { /// Creates type info for an iterator. /// /// Iterator type id, such as . - /// Python interpreter - public PythonIteratorType(BuiltinTypeId typeId, IPythonInterpreter interpreter) - : base(typeId, interpreter.ModuleResolution.BuiltinsModule) { + /// Declaring module. + public PythonIteratorType(BuiltinTypeId typeId, IPythonModule declaringModule) + : base(typeId, declaringModule) { TypeId = typeId; } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassMember.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassMember.cs index 2f95d489a..9ea3e402f 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassMember.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassMember.cs @@ -17,7 +17,7 @@ namespace Microsoft.Python.Analysis.Types { /// /// Represents member of a class. /// - public interface IPythonClassMember : IPythonType, ILocatedMember { + public interface IPythonClassMember : IPythonType { IPythonType DeclaringType { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs index 32ae2380c..2079cb2e2 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs @@ -21,7 +21,7 @@ namespace Microsoft.Python.Analysis.Types { /// /// Represents Python class type definition. /// - public interface IPythonClassType : IPythonType, IGenericType, ILocatedMember { + public interface IPythonClassType : IPythonClassMember, IGenericType { /// /// Class definition node in the AST. /// @@ -41,6 +41,6 @@ public interface IPythonClassType : IPythonType, IGenericType, ILocatedMember { /// If class is created off generic template, represents /// pairs of the generic parameter / actual supplied type. /// - IReadOnlyDictionary GenericParameters { get; } + IReadOnlyDictionary GenericParameters { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs index 000fa0987..115c6168b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonCollectionType.cs @@ -19,7 +19,7 @@ namespace Microsoft.Python.Analysis.Types { /// /// Represents instance of a collection. /// - public interface IPythonCollectionType : IPythonType { + public interface IPythonCollectionType : IPythonClassType { /// /// Type of the collection iterator. /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs index f71505cb8..2f8706a8e 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs @@ -53,7 +53,6 @@ public interface IPythonFunctionOverload { /// Call arguments or type arguments. /// Invoking class instance. In case of generics it is instance of the specific type /// as opposed to declaring type which is the generic template class. - /// Call expression location, if any. IMember Call(IArgumentSet args, IPythonType self); /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs index ae86bc87f..168670be1 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs @@ -13,8 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Threading; -using System.Threading.Tasks; +using System; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; @@ -22,7 +21,17 @@ namespace Microsoft.Python.Analysis.Types { /// /// Represents a Python module. /// - public interface IPythonModule : IPythonType, IPythonFile, ILocatedMember { + public interface IPythonModule : IPythonType { + /// + /// File path to the module. + /// + string FilePath { get; } + + /// + /// Module URI. + /// + Uri Uri { get; } + /// /// Module analysis. /// @@ -54,5 +63,10 @@ public interface IPythonModule : IPythonType, IPythonFile, ILocatedMember { /// wants to see library code and not a stub. /// IPythonModule PrimaryModule { get; } + + /// + /// Indicates if module is restored from database. + /// + bool IsPersistent { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs index 20fc3c932..badcecb6b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs @@ -34,5 +34,10 @@ public interface IPythonPropertyType : IPythonClassMember { /// True if the property is read-only. /// bool IsReadOnly { get; } + + /// + /// Property return type. + /// + IMember ReturnType { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs index 00333da4a..d90ec08d7 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonType.cs @@ -25,6 +25,11 @@ public interface IPythonType : ILocatedMember, IMemberContainer { /// string Name { get; } + /// + /// Fully qualified type name. Used for analysis persistence. + /// + string QualifiedName { get; } + /// /// Indicates built-in type id such as 'int' or 'str' /// or 'type' for user-defined entities. diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 36899d865..2a70b4004 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -54,6 +54,9 @@ public virtual IReadOnlyList References { public virtual void AddReference(Location location) { lock (this) { + if(this.DeclaringModule == null || this.DeclaringModule?.ModuleType == ModuleType.Builtins) { + return; + } // Don't add references to library code. if (location.Module?.ModuleType == ModuleType.User && !location.Equals(Location)) { _references = _references ?? new HashSet(); diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index 2485d307d..e7f49e0bd 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -30,9 +30,8 @@ public Location(IPythonModule module, IndexSpan indexSpan) { public LocationInfo LocationInfo { get { - var ast = Module?.GetAst(); - if (ast != null && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) { - return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(ast)); + if (Module is ILocationConverter lc && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) { + return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(lc)); } return LocationInfo.Empty; } diff --git a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs index 07f5a5e5b..c03e8b56a 100644 --- a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs @@ -29,7 +29,7 @@ public ParameterInfo(PythonAst ast, Parameter p, IPythonType type, IMember defau } public ParameterInfo(string name, IPythonType type, ParameterKind? kind, IMember defaultValue) { - Name = name ?? throw new ArgumentNullException(nameof(name)); + Name = name ?? "*"; // ?? throw new ArgumentNullException(nameof(name)); Documentation = string.Empty; DefaultValue = defaultValue; Type = type; diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index c64a673ef..9490079e6 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -26,14 +26,16 @@ internal partial class PythonClassType { private readonly ReentrancyGuard _genericSpecializationGuard = new ReentrancyGuard(); private readonly ReentrancyGuard _genericResolutionGuard = new ReentrancyGuard(); - private bool _isGeneric; + private string _nameWithParameters; // Name of the class with generic parameters like abc[int]. + private string _qualifiedNameWithParameters; // Qualified name with qualified parameter names for persistence. + private Dictionary _specificTypeCache; - private Dictionary _genericParameters; + private Dictionary _genericParameters; private IReadOnlyList _parameters = new List(); #region IGenericType /// - /// List of unfilled generic type parameters. Represented as entries in the GenericParameters dictionary + /// List of unfilled generic type parameters. Represented as entries in the ActualGenericParameters dictionary /// that have both key and value as generic type parameters /// e.g /// {T, T} @@ -41,11 +43,9 @@ internal partial class PythonClassType { /// public virtual IReadOnlyList Parameters => _parameters ?? Array.Empty(); - public virtual bool IsGeneric => _isGeneric; - + public virtual bool IsGeneric { get; private set; } #endregion - #region IPythonTemplateType /// /// Given an argument set with types, e.g int, float, str, look at the generic type parameters for the class /// and create a new class type with those parameters filled in. Additionally, transmit the specific class @@ -58,7 +58,7 @@ internal partial class PythonClassType { /// B[int] defines the type parameter T to be of type int and type parameter U to be type str. /// B[int] inherits from A[int, str] /// - public IPythonType CreateSpecificType(IArgumentSet args) { + public virtual IPythonType CreateSpecificType(IArgumentSet args) { lock (_membersLock) { var newGenericTypeParameters = GetTypeParameters(); var newBases = new List(); @@ -68,11 +68,15 @@ public IPythonType CreateSpecificType(IArgumentSet args) { var genericTypeToSpecificType = GetSpecificTypes(args, newGenericTypeParameters, newBases); var classType = new PythonClassType(BaseName, new Location(DeclaringModule)); + classType.SetDocumentation(Documentation); + // Storing generic parameters allows methods returning generic types // to know what type parameter returns what specific type - classType.StoreGenericParameters(this, newGenericTypeParameters, genericTypeToSpecificType); + classType.StoreGenericParameters(this, newGenericTypeParameters.Select(p => p.Name).ToArray(), genericTypeToSpecificType); // Set generic name + classType.SetNames(); + // Locking so threads can only access class after it's been initialized // Store generic parameters first so name updates correctly, then check if class type has been cached _specificTypeCache = _specificTypeCache ?? new Dictionary(); @@ -81,7 +85,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { } _specificTypeCache[classType.Name] = classType; - // Prevent reentrancy when resolving generic class where method may be returning instance of type of the same class. + // Prevent re-entrancy when resolving generic class where method may be returning instance of type of the same class. // e.g // class C(Generic[T]): // def tmp(self): @@ -96,7 +100,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { // Bases can be null when not set var bases = Bases ?? Array.Empty(); // Get declared generic class parameters, i.e. Generic[T1, T2, ...], Optional[Generic[T1, ...]] - var genericClassParameters = bases.OfType().ToArray(); + var genericClassParameters = bases.OfType().ToArray(); // Get list of bases that are generic but not generic class parameters, e.g A[T], B[T] but not Generic[T1, T2] var genericTypeBases = bases.Except(genericClassParameters).OfType().Where(g => g.IsGeneric).ToArray(); @@ -111,7 +115,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { // Look through generic type bases and see if any of their required type parameters // have received a specific type, and if so create specific type var st = gt.Parameters - .Select(p => classType.GenericParameters.TryGetValue(p, out var t) ? t : null) + .Select(p => classType.GenericParameters.TryGetValue(p.Name, out var t) ? t : null) .Where(p => !p.IsUnknown()) .ToArray(); if (st.Length > 0) { @@ -123,10 +127,9 @@ public IPythonType CreateSpecificType(IArgumentSet args) { } // Set specific class bases - classType.SetBases(specificBases.Concat(newBases), args.Eval.CurrentScope); + classType.SetBases(specificBases.Concat(newBases), args.Eval?.CurrentScope); // Now that parameters are set, check if class is generic - classType._parameters = classType._genericParameters.Values.Distinct().OfType().ToList(); - classType.DecideGeneric(); + classType.SetGenericParameters(); // Transfer members from generic to specific type. classType.SetClassMembers(this, args); } @@ -134,8 +137,6 @@ public IPythonType CreateSpecificType(IArgumentSet args) { } } - #endregion - /// /// Gets a list of distinct type parameters from bases and the class itself /// @@ -147,7 +148,7 @@ private IGenericTypeParameter[] GetTypeParameters() { var bases = Bases ?? Array.Empty(); var fromBases = new HashSet(); - var genericClassParameter = bases.OfType().FirstOrDefault(); + var genericClassParameter = bases.OfType().FirstOrDefault(); // If Generic[...] is present, ordering of type variables is determined from that if (genericClassParameter?.TypeParameters != null) { @@ -167,13 +168,13 @@ private IGenericTypeParameter[] GetTypeParameters() { /// Given an argument set, returns a dictionary mapping generic type parameter to the supplied specific /// type from arguments. /// - private IReadOnlyDictionary GetSpecificTypes( + private IReadOnlyDictionary GetSpecificTypes( IArgumentSet args, IReadOnlyList genericTypeParameters, ICollection newBases ) { // For now, map each type parameter to itself, and we can fill in the value as we go - var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp, gtp => gtp as IPythonType); + var genericTypeToSpecificType = genericTypeParameters.ToDictionary(gtp => gtp.Name, gtp => gtp as IPythonType); // Arguments passed are those of __init__ or copy constructor or index expression A[int, str, ...]. // The arguments do not necessarily match all of the declared generic parameters. @@ -189,11 +190,11 @@ ICollection newBases // __init__(self, v: _T), v is annotated as a generic type definition // Check if its generic type name matches any of the generic class parameters i.e. if there is // an argument like 'v: _T' we need to check if class has matching Generic[_T] or A[_T] in bases. - if (genericTypeToSpecificType.ContainsKey(argTypeDefinition)) { + if (genericTypeToSpecificType.ContainsKey(argTypeDefinition.Name)) { // TODO: Check if specific type matches generic type parameter constraints and report mismatches. // Assign specific type. if (arg.Value is IMember m && m.GetPythonType() is IPythonType pt) { - genericTypeToSpecificType[argTypeDefinition] = pt; + genericTypeToSpecificType[argTypeDefinition.Name] = pt; } else { // TODO: report supplied parameter is not a type. } @@ -204,7 +205,7 @@ ICollection newBases } // Don't add generic type parameters to bases - if (!(arg.Value is IGenericTypeParameter) && arg.Value is IMember member && !member.GetPythonType().IsUnknown()) { + if (_bases != null && !(arg.Value is IGenericTypeParameter) && arg.Value is IMember member && !member.GetPythonType().IsUnknown()) { var type = member.GetPythonType(); // Type may be a specific type created off generic or just a type // for the copy constructor. Consider 'class A(Generic[K, V], Mapping[K, V])' @@ -240,8 +241,8 @@ ICollection newBases var type = member.GetPythonType(); if (!type.IsUnknown()) { var gtd = gtIndex < genericTypeParameters.Count ? genericTypeParameters[gtIndex] : null; - if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd, out var s) && s is IGenericTypeParameter) { - genericTypeToSpecificType[gtd] = type; + if (gtd != null && genericTypeToSpecificType.TryGetValue(gtd.Name, out var s) && s is IGenericTypeParameter) { + genericTypeToSpecificType[gtd.Name] = type; } gtIndex++; } @@ -252,13 +253,13 @@ ICollection newBases } /// - /// Points the generic type parameter in class type to their corresponding specific type (or a generic - /// type parameter if no specific type was provided) + /// Points the generic type parameter in class type to their corresponding specific type + /// (or a generic type parameter if no specific type was provided) /// - private void StoreGenericParameters( + internal void StoreGenericParameters( IPythonClassType templateClass, - IEnumerable newGenericParameters, - IReadOnlyDictionary genericToSpecificTypes) { + IEnumerable newGenericParameters, + IReadOnlyDictionary genericToSpecificTypes) { // copy original generic parameters over and try to fill them in _genericParameters = templateClass.GenericParameters.ToDictionary(k => k.Key, k => k.Value); @@ -279,14 +280,12 @@ private void StoreGenericParameters( // class A(Generic[T]): // class B(A[U]) // A has T => U - _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType, out var v) ? v : null; + _genericParameters[gp] = genericToSpecificTypes.TryGetValue(specificType.Name, out var v) ? v : null; } } } - if (!_genericParameters.IsNullOrEmpty()) { - _genericName = CodeFormatter.FormatSequence(BaseName, '[', _genericParameters.Values); - } + SetNames(); } /// @@ -303,28 +302,28 @@ private void StoreGenericParameters( /// Generic type (Generic[T1, T2, ...], A[T1, T2, ..], etc.). /// Argument value passed to the class constructor. /// Dictionary or name (T1) to specific type to populate. - private static void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) { + private void GetSpecificTypeFromArgumentValue(IGenericType gt, object argumentValue, IDictionary specificTypes) { switch (argumentValue) { case IPythonDictionary dict when gt.Parameters.Count == 2: var keyType = dict.Keys.FirstOrDefault()?.GetPythonType(); var valueType = dict.Values.FirstOrDefault()?.GetPythonType(); if (!keyType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = keyType; + specificTypes[gt.Parameters[0].Name] = keyType; } if (!valueType.IsUnknown()) { - specificTypes[gt.Parameters[1]] = valueType; + specificTypes[gt.Parameters[1].Name] = valueType; } break; case IPythonIterable iter when gt.TypeId == BuiltinTypeId.List && gt.Parameters.Count == 1: var itemType = iter.GetIterator().Next.GetPythonType(); if (!itemType.IsUnknown()) { - specificTypes[gt.Parameters[0]] = itemType; + specificTypes[gt.Parameters[0].Name] = itemType; } break; case IPythonCollection coll when gt.TypeId == BuiltinTypeId.Tuple && gt.Parameters.Count >= 1: var itemTypes = coll.Contents.Select(m => m.GetPythonType()).ToArray(); for (var i = 0; i < Math.Min(itemTypes.Length, gt.Parameters.Count); i++) { - specificTypes[gt.Parameters[i]] = itemTypes[i]; + specificTypes[gt.Parameters[i].Name] = itemTypes[i]; } break; } @@ -362,7 +361,7 @@ private void SetClassMembers(IPythonClassType templateClass, IArgumentSet args) specificType = tt.CreateSpecificType(args); break; case IGenericTypeParameter gtd: - GenericParameters.TryGetValue(gtd, out specificType); + GenericParameters.TryGetValue(gtd.Name, out specificType); break; } @@ -379,12 +378,26 @@ private void SetClassMembers(IPythonClassType templateClass, IArgumentSet args) /// Determines if the class is generic. /// A class is generic if it has at least one unfilled generic type parameters or one of its bases is generic /// - public void DecideGeneric() { + private void DecideGeneric() { using (_genericResolutionGuard.Push(this, out var reentered)) { if (!reentered) { - _isGeneric = !Parameters.IsNullOrEmpty() || (Bases?.OfType().Any(g => g.IsGeneric) ?? false); + IsGeneric = !Parameters.IsNullOrEmpty() || (Bases?.OfType().Any(g => g.IsGeneric) ?? false); } } } + + private void SetNames() { + // Based on available data, calculate name of generic with parameters, if any, + // as well as qualified name. + if (!_genericParameters.IsNullOrEmpty()) { + _nameWithParameters = CodeFormatter.FormatSequence(BaseName, '[', _genericParameters.Values); + _qualifiedNameWithParameters = CodeFormatter.FormatSequence(BaseName, '[', _genericParameters.Values.Select(v => v.QualifiedName)); + } + } + + private void SetGenericParameters() { + _parameters = _genericParameters.Values.Distinct().OfType().ToList(); + DecideGeneric(); + } } } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 457952d11..0f35ff1d9 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -25,19 +25,22 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; -using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { - [DebuggerDisplay("Class {Name}")] + [DebuggerDisplay("Class {" + nameof(Name) + "}")] internal partial class PythonClassType : PythonType, IPythonClassType, IEquatable { + internal enum ClassDocumentationSource { + Class, + Init, + Base + } private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); private readonly object _membersLock = new object(); - private string _genericName; private List _bases; private IReadOnlyList _mro; private string _documentation; @@ -47,18 +50,22 @@ internal PythonClassType(string name, Location location) : base(name, location, string.Empty, BuiltinTypeId.Type) { } - public PythonClassType(ClassDefinition classDefinition, Location location, BuiltinTypeId builtinTypeId = BuiltinTypeId.Type) - : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { - Check.ArgumentNotNull(nameof(location), location.Module); + public PythonClassType( + ClassDefinition classDefinition, + IPythonType declaringType, + Location location, + BuiltinTypeId builtinTypeId = BuiltinTypeId.Type + ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { location.Module.AddAstNode(this, classDefinition); + DeclaringType = declaringType; } + #region IPythonType /// /// If class has generic type parameters, returns that form, e.g 'A[T1, int, ...]', otherwise returns base, e.g 'A' /// - public override string Name => _genericName ?? base.Name; - - #region IPythonType + public override string Name => _nameWithParameters ?? base.Name; + public override string QualifiedName => this.GetQualifiedName(_qualifiedNameWithParameters); public override PythonMemberType MemberType => PythonMemberType.Class; public override IEnumerable GetMemberNames() { @@ -85,7 +92,7 @@ public override IMember GetMember(string name) { switch (name) { case "__mro__": case "mro": - return is3x ? PythonCollectionType.CreateList(DeclaringModule.Interpreter, Mro) : UnknownType as IMember; + return is3x ? PythonCollectionType.CreateList(DeclaringModule, Mro) : UnknownType as IMember; case "__dict__": return is3x ? DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Dict) : UnknownType; case @"__weakref__": @@ -118,18 +125,23 @@ public override string Documentation { } // Try doc from the type first (class definition AST node). _documentation = base.Documentation; + DocumentationSource = ClassDocumentationSource.Class; + if (string.IsNullOrEmpty(_documentation)) { // If not present, try docs __init__. IPythonFunctionType handles // __init__ in a special way so there is no danger of call coming // back here and causing stack overflow. _documentation = (GetMember("__init__") as IPythonFunctionType)?.Documentation; + DocumentationSource = ClassDocumentationSource.Init; } if (string.IsNullOrEmpty(_documentation) && Bases != null) { // If still not found, try bases. var o = DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - _documentation = Bases.FirstOrDefault(b => b != o && !string.IsNullOrEmpty(b?.Documentation)) - ?.Documentation; + _documentation = Bases + .FirstOrDefault(b => b != o && !(b is IGenericClassBase) && !string.IsNullOrEmpty(b?.Documentation))? + .Documentation; + DocumentationSource = ClassDocumentationSource.Base; } } return _documentation; @@ -138,18 +150,19 @@ public override string Documentation { // Constructor call public override IMember CreateInstance(string typeName, IArgumentSet args) { + var builtins = DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule; // Specializations switch (typeName) { case "list": - return PythonCollectionType.CreateList(DeclaringModule.Interpreter, args); + return PythonCollectionType.CreateList(builtins, args); case "dict": { // self, then contents var contents = args.Values().Skip(1).FirstOrDefault(); - return new PythonDictionary(DeclaringModule.Interpreter, contents); + return new PythonDictionary(builtins, contents); } case "tuple": { var contents = args.Values(); - return PythonCollectionType.CreateTuple(DeclaringModule.Interpreter, contents); + return PythonCollectionType.CreateTuple(builtins, contents); } } return new PythonInstance(this); @@ -167,6 +180,10 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { } #endregion + #region IPythonClassMember + public IPythonType DeclaringType { get; } + #endregion + #region IPythonClass public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases { @@ -182,7 +199,7 @@ public IReadOnlyList Mro { if (_mro != null) { return _mro; } - if (_bases == null) { + if (_bases == null || _bases.Count == 0) { return new IPythonType[] { this }; } _mro = new IPythonType[] { this }; @@ -197,11 +214,25 @@ public IReadOnlyList Mro { /// class B(A[int, str]): ... /// Has the map {T: int, K: str} /// - public virtual IReadOnlyDictionary GenericParameters => - _genericParameters ?? EmptyDictionary.Instance; + public virtual IReadOnlyDictionary GenericParameters => + _genericParameters ?? EmptyDictionary.Instance; #endregion + internal ClassDocumentationSource DocumentationSource { get; private set; } + + internal override void SetDocumentation(string documentation) { + _documentation = documentation; + DocumentationSource = ClassDocumentationSource.Class; + } + + /// + /// Sets class bases. If scope is provided, detects loops in base classes and removes them. + /// + /// List of base types. + /// Current scope to look up base types. + /// Can be null if class is restored from database, in which case + /// there is no need to try and disambiguate bases. internal void SetBases(IEnumerable bases, IScope currentScope = null) { if (_bases != null) { return; // Already set @@ -229,12 +260,14 @@ internal void SetBases(IEnumerable bases, IScope currentScope = nul } // Invalidate MRO _mro = null; + DecideGeneric(); + if (DeclaringModule is BuiltinsPythonModule) { // TODO: If necessary, we can set __bases__ on builtins when the module is fully analyzed. return; } - AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter, _bases), true); + AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule, _bases), true); } /// diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 4a94c0e5d..e3158c24c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { @@ -30,18 +31,19 @@ namespace Microsoft.Python.Analysis.Types { /// Module making the call. /// Function overload the return value is requested for. /// Call arguments. - /// + /// Location of the call expression. public delegate IMember ReturnValueProvider( IPythonModule declaringModule, IPythonFunctionOverload overload, - IArgumentSet args); + IArgumentSet args, + IndexSpan indexSpan); internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOverload { private readonly string _returnDocumentation; // Allow dynamic function specialization, such as defining return types for builtin // functions that are impossible to scrape and that are missing from stubs. - // Parameters: declaring module, overload for the return value, list of arguments. + // FormalGenericParameters: declaring module, overload for the return value, list of arguments. private ReturnValueProvider _returnValueProvider; // Return value can be an instance or a type info. Consider type(C()) returning @@ -49,15 +51,15 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve private bool _fromAnnotation; public PythonFunctionOverload(IPythonClassMember cm, FunctionDefinition fd, Location location, string returnDocumentation) - : this(cm.Name, location) { - ClassMember = cm; + : this(cm, location) { Documentation = fd.GetDocumentation(); cm.DeclaringModule.AddAstNode(this, fd); _returnDocumentation = returnDocumentation; } - public PythonFunctionOverload(string name, Location location) : base(location) { - Name = name ?? throw new ArgumentNullException(nameof(name)); + public PythonFunctionOverload(IPythonClassMember cm, Location location) : base(location) { + ClassMember = cm ?? throw new ArgumentNullException(nameof(cm)); + Name = cm.Name ?? throw new ArgumentNullException(nameof(cm.Name)); } #region ILocatedMember @@ -122,7 +124,7 @@ public string GetReturnDocumentation(IPythonType self = null) { public IMember Call(IArgumentSet args, IPythonType self) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(DeclaringModule, this, args); + var rt = _returnValueProvider?.Invoke(args.Eval.Module, this, args, default); if (!rt.IsUnknown()) { return rt; } @@ -159,7 +161,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType // -> A[_T1, _T2, ...] // Match arguments IReadOnlyList typeArgs = null; - var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty(); + var classGenericParameters = selfClassType?.GenericParameters.Keys.ToArray() ?? Array.Empty(); if (classGenericParameters.Length > 0 && selfClassType != null) { // Declaring class is specific and provides definitions of generic parameters typeArgs = classGenericParameters @@ -179,7 +181,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType } private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType, IArgumentSet args, IGenericTypeParameter returnType) { - if (selfClassType.GetSpecificType(returnType, out var specificType)) { + if (selfClassType.GetSpecificType(returnType.Name, out var specificType)) { return new PythonInstance(specificType); } @@ -187,16 +189,15 @@ private IMember CreateSpecificReturnFromTypeVar(IPythonClassType selfClassType, var baseType = selfClassType.Mro .OfType() .Skip(1) - .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType)); + .FirstOrDefault(b => b.GetMember(ClassMember.Name) != null && b.GenericParameters.ContainsKey(returnType.Name)); // Try and infer return value from base class - if (baseType != null && baseType.GetSpecificType(returnType, out specificType)) { + if (baseType != null && baseType.GetSpecificType(returnType.Name, out specificType)) { return new PythonInstance(specificType); } // Try getting type from passed in arguments - var typeFromArgs = args?.Arguments.FirstOrDefault(a => returnType.Equals(a.Type))?.Value as IMember; - if (typeFromArgs != null) { + if (args?.Arguments.FirstOrDefault(a => returnType.Equals(a.Type))?.Value is IMember typeFromArgs) { return typeFromArgs; } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index c0c9a0e43..044f6c5c0 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; @@ -50,7 +51,7 @@ public PythonFunctionType( Location location, IPythonType declaringType, string documentation - ) : base(name, location, documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + ) : base(name, location, documentation, declaringType is IPythonClassType ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; } @@ -60,8 +61,13 @@ public PythonFunctionType( Location location ) : base(fd.Name, location, fd.Name == "__init__" ? (declaringType?.Documentation ?? fd.GetDocumentation()) : fd.GetDocumentation(), - declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + declaringType is IPythonClassType ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; + + // IsStub must be set permanently so when location of the stub is reassigned + // to the primary module for navigation purposes, function still remembers + // that it case from a stub. + IsStub = location.Module.ModuleType == ModuleType.Stub; location.Module.AddAstNode(this, fd); ProcessDecorators(fd); @@ -71,6 +77,8 @@ Location location public override PythonMemberType MemberType => TypeId == BuiltinTypeId.Function ? PythonMemberType.Function : PythonMemberType.Method; + public override string QualifiedName => this.GetQualifiedName(); + public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) { // Now we can go and find overload with matching arguments. var overload = Overloads[args.OverloadIndex]; @@ -96,7 +104,7 @@ internal override void SetDocumentation(string documentation) { public override bool IsSpecialized => _isSpecialized; public bool IsOverload { get; private set; } - public bool IsStub { get; internal set; } + public bool IsStub { get; } public bool IsUnbound => DeclaringType == null; public IReadOnlyList Overloads => _overloads; @@ -114,7 +122,10 @@ internal void Specialize(string[] dependencies) { internal void AddOverload(IPythonFunctionOverload overload) => _overloads = _overloads.Count > 0 ? _overloads.Add(overload) : ImmutableArray.Create(overload); - internal IPythonFunctionType ToUnbound() => new PythonUnboundMethod(this); + internal IPythonFunctionType ToUnbound() { + Debug.Assert(DeclaringType != null, "Attempt to unbound standalone function."); + return new PythonUnboundMethod(this); + } private void ProcessDecorators(FunctionDefinition fd) { foreach (var dec in (fd.Decorators?.Decorators).MaybeEnumerate().OfType()) { @@ -153,22 +164,21 @@ private void ProcessDecorators(FunctionDefinition fd) { /// /// Represents unbound method, such in C.f where C is class rather than the instance. /// - private sealed class PythonUnboundMethod : PythonTypeWrapper, IPythonFunctionType { - private readonly IPythonFunctionType _pf; - + internal sealed class PythonUnboundMethod : PythonTypeWrapper, IPythonFunctionType { public PythonUnboundMethod(IPythonFunctionType function) : base(function, function.DeclaringModule) { - _pf = function; + Function = function; } - public FunctionDefinition FunctionDefinition => _pf.FunctionDefinition; - public IPythonType DeclaringType => _pf.DeclaringType; - public bool IsStatic => _pf.IsStatic; - public bool IsClassMethod => _pf.IsClassMethod; - public bool IsOverload => _pf.IsOverload; - public bool IsStub => _pf.IsStub; + public IPythonFunctionType Function { get; } + public FunctionDefinition FunctionDefinition => Function.FunctionDefinition; + public IPythonType DeclaringType => Function.DeclaringType; + public bool IsStatic => Function.IsStatic; + public bool IsClassMethod => Function.IsClassMethod; + public bool IsOverload => Function.IsOverload; + public bool IsStub => Function.IsStub; public bool IsUnbound => true; - public IReadOnlyList Overloads => _pf.Overloads; + public IReadOnlyList Overloads => Function.Overloads; public override BuiltinTypeId TypeId => BuiltinTypeId.Function; public override PythonMemberType MemberType => PythonMemberType.Function; } diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 9728627fa..12297dace 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -18,22 +18,23 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { - class PythonPropertyType : PythonType, IPythonPropertyType { + internal sealed class PythonPropertyType : PythonType, IPythonPropertyType { private IPythonFunctionOverload _getter; public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) - : this(fd.Name, location, declaringType, isAbstract) { + : this(fd.Name, location, fd.GetDocumentation(), declaringType, isAbstract) { declaringType.DeclaringModule.AddAstNode(this, fd); } - public PythonPropertyType(string name, Location location, IPythonType declaringType, bool isAbstract) - : base(name, location, string.Empty, BuiltinTypeId.Property) { + public PythonPropertyType(string name, Location location, string documentation, IPythonType declaringType, bool isAbstract) + : base(name, location, documentation, BuiltinTypeId.Property) { DeclaringType = declaringType; IsAbstract = isAbstract; } #region IPythonType public override PythonMemberType MemberType => PythonMemberType.Property; + public override string QualifiedName => this.GetQualifiedName(); #endregion #region IPythonPropertyType @@ -41,13 +42,20 @@ public PythonPropertyType(string name, Location location, IPythonType declaringT public override bool IsAbstract { get; } public bool IsReadOnly => true; public IPythonType DeclaringType { get; } - public string Description - => Type == null ? Resources.PropertyOfUnknownType : Resources.PropertyOfType.FormatUI(Type.Name); + + public string Description { + get { + var typeName = ReturnType?.GetPythonType()?.Name; + return typeName != null ? Resources.PropertyOfType.FormatUI(typeName) : Resources.PropertyOfUnknownType; + } + } + public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => _getter?.Call(args, instance?.GetPythonType() ?? DeclaringType); + => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); + + public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType); #endregion internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; - private IPythonType Type => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType)?.GetPythonType(); } } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 7d5f0fb67..8058a08c8 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -22,8 +22,8 @@ using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { - [DebuggerDisplay("{Name}")] - internal class PythonType : LocatedMember, IPythonType {//, IEquatable { + [DebuggerDisplay("{" + nameof(Name) + "}")] + internal class PythonType : LocatedMember, IPythonType { private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; @@ -59,6 +59,11 @@ public override void AddReference(Location location) { #region IPythonType + public virtual string QualifiedName + => DeclaringModule.ModuleType == ModuleType.Builtins + ? TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : Name + : this.GetQualifiedName(); + public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : BaseName; public virtual string Documentation { get; private set; } public virtual BuiltinTypeId TypeId => _typeId; @@ -154,16 +159,14 @@ internal IMember AddMember(string name, IMember member, bool overwrite) { } } - internal void MakeReadOnly() => _readonly = true; + internal void MakeReadOnly() { + lock (_lock) { + _readonly = true; + } + } internal bool IsHidden => ContainsMember("__hidden__"); protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; - - //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); - - //public override bool Equals(object obj) - // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); - //public override int GetHashCode() => 0; } } diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index d67fa2358..f24ffa4a9 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Types { /// @@ -24,6 +23,8 @@ namespace Microsoft.Python.Analysis.Types { /// internal class PythonTypeWrapper : IPythonType { private readonly BuiltinTypeId _builtinTypeId; + private readonly string _typeName; + private readonly string _documentation; private IPythonType _innerType; protected IPythonType InnerType @@ -33,7 +34,11 @@ protected IPythonType InnerType /// Creates delegate type wrapper over an existing type. /// Use dedicated constructor for wrapping builtin types. /// - public PythonTypeWrapper(IPythonType type) : this(type, type.DeclaringModule) { + public PythonTypeWrapper(IPythonType type) : this(type, type.DeclaringModule) { } + + public PythonTypeWrapper(string typeName, string documentation, IPythonModule declaringModule, IPythonType baseType) : this(baseType, declaringModule) { + _typeName = typeName; + _documentation = documentation; } /// @@ -56,9 +61,10 @@ public PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaringMod } #region IPythonType - public virtual string Name => InnerType.Name; + public virtual string Name => _typeName ?? InnerType.Name; + public virtual string QualifiedName => _typeName != null ? $"{DeclaringModule.Name}:{_typeName}" : InnerType.QualifiedName; public IPythonModule DeclaringModule { get; } - public virtual string Documentation => InnerType.Documentation; + public virtual string Documentation => _documentation ?? InnerType.Documentation; public virtual BuiltinTypeId TypeId => InnerType.TypeId; public virtual PythonMemberType MemberType => InnerType.MemberType; public virtual bool IsBuiltin => InnerType.IsBuiltin; @@ -74,11 +80,11 @@ public virtual IMember Index(IPythonInstance instance, IArgumentSet args) #endregion #region ILocatedMember - public Location Location => InnerType?.Location ?? default; - public LocationInfo Definition => InnerType?.Definition ?? LocationInfo.Empty; - public IReadOnlyList References => InnerType?.References ?? Array.Empty(); - public void AddReference(Location location) => InnerType?.AddReference(location); - public void RemoveReferences(IPythonModule module) => InnerType?.RemoveReferences(module); + public virtual Location Location => InnerType?.Location ?? default; + public virtual LocationInfo Definition => InnerType?.Definition ?? LocationInfo.Empty; + public virtual IReadOnlyList References => InnerType?.References ?? Array.Empty(); + public virtual void AddReference(Location location) => InnerType?.AddReference(location); + public virtual void RemoveReferences(IPythonModule module) => InnerType?.RemoveReferences(module); #endregion #region IMemberContainer diff --git a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs index 7f6ff4dce..005268134 100644 --- a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs @@ -26,11 +26,13 @@ internal sealed class PythonUnionType : LocatedMember, IPythonUnionType { private readonly HashSet _types = new HashSet(PythonTypeComparer.Instance); private readonly object _lock = new object(); - public PythonUnionType(IEnumerable types, IPythonModule declaringModule) : base(declaringModule) { + public PythonUnionType(IEnumerable types, IPythonModule declaringModule) + : base(declaringModule.Interpreter.ModuleResolution.GetSpecializedModule("typing")) { _types.UnionWith(types); } - private PythonUnionType(IPythonType x, IPythonType y) : base(x.DeclaringModule) { + private PythonUnionType(IPythonType x, IPythonType y) + : base(x.DeclaringModule.Interpreter.ModuleResolution.GetSpecializedModule("typing")) { Check.Argument(nameof(x), () => !(x is IPythonUnionType)); Check.Argument(nameof(y), () => !(y is IPythonUnionType)); _types.Add(x); @@ -49,8 +51,12 @@ public string Name { } } - public override IPythonModule DeclaringModule { - get { lock (_lock) { return _types.First().DeclaringModule; } } + public string QualifiedName { + get { + lock (_lock) { + return CodeFormatter.FormatSequence("typing:Union", '[', _types.Select(t => t.QualifiedName).ToArray()); + } + } } public BuiltinTypeId TypeId => BuiltinTypeId.Type; diff --git a/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs b/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs index 82cf7835b..8adbd2782 100644 --- a/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs +++ b/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs @@ -19,7 +19,7 @@ using Microsoft.Python.Core.Disposables; namespace Microsoft.Python.Analysis.Utilities { - public sealed class ReentrancyGuard { + internal sealed class ReentrancyGuard { private readonly AsyncLocal> _stack = new AsyncLocal>(); public IDisposable Push(T t, out bool reentered) { @@ -40,7 +40,7 @@ public IDisposable Push(T t, out bool reentered) { public void Pop() { _stack.Value.Pop(); - if(_stack.Value.Count == 0) { + if (_stack.Value.Count == 0) { _stack.Value = null; } } diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs index 1e8da8abb..ab49f8193 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonCollection.cs @@ -25,7 +25,7 @@ internal class PythonCollection : PythonInstance, IPythonCollection { /// Collection type. /// Contents of the collection (typically elements from the initialization). /// If true and contents is a single element - /// True if the contents are an exact representation of the collection contents. + /// True if the contents are an exact representation of the collection contents. /// and is a sequence, the sequence elements are copied rather than creating /// a sequence of sequences with a single element. public PythonCollection( diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs index 163015002..53fc8175d 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs @@ -36,27 +36,27 @@ public PythonDictionary(PythonDictionaryType dictType, IReadOnlyDictionary(), exact: exact) { + public PythonDictionary(IPythonModule declaringModule, IMember contents, bool exact = false) : + base(new PythonDictionaryType(declaringModule), Array.Empty(), exact: exact) { if (contents is IPythonDictionary dict) { foreach (var key in dict.Keys) { _contents[key] = dict[key]; } Contents = _contents.Keys.ToArray(); } - _interpreter = interpreter; + _interpreter = declaringModule.Interpreter; } - public PythonDictionary(IPythonInterpreter interpreter, IReadOnlyDictionary contents, bool exact = false) : - this(new PythonDictionaryType(interpreter), contents, exact: exact) { - _interpreter = interpreter; + public PythonDictionary(IPythonModule declaringModule, IReadOnlyDictionary contents, bool exact = false) : + this(new PythonDictionaryType(declaringModule), contents, exact: exact) { + _interpreter = declaringModule.Interpreter; } public IEnumerable Keys => _contents.Keys.ToArray(); public IEnumerable Values => _contents.Values.ToArray(); public IReadOnlyList Items - => _contents.Select(kvp => PythonCollectionType.CreateTuple(Type.DeclaringModule.Interpreter, new[] { kvp.Key, kvp.Value })).ToArray(); + => _contents.Select(kvp => PythonCollectionType.CreateTuple(Type.DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule, new[] { kvp.Key, kvp.Value })).ToArray(); public IMember this[IMember key] => _contents.TryGetValue(key, out var value) ? value : UnknownType; @@ -103,7 +103,7 @@ public override IMember Call(string memberName, IArgumentSet args) { } private IPythonCollection CreateList(IReadOnlyList items) - => PythonCollectionType.CreateList(Type.DeclaringModule.Interpreter, items, false); + => PythonCollectionType.CreateList(Type.DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule, items, false); private sealed class KeyComparer : IEqualityComparer { public bool Equals(IMember x, IMember y) { diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs index 6b7216e93..2cc207822 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs @@ -25,7 +25,7 @@ internal sealed class PythonInstanceIterator : PythonInstance, IPythonIterator { private readonly IPythonFunctionType __next__; public PythonInstanceIterator(IMember instance, IPythonInterpreter interpreter) - : base(new PythonIteratorType(BuiltinTypeId.SetIterator, interpreter)) { + : base(new PythonIteratorType(BuiltinTypeId.SetIterator, interpreter.ModuleResolution.BuiltinsModule)) { __next__ = instance.GetPythonType().GetMember(@"__next__") as IPythonFunctionType; } diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonIterator.cs index e97263f34..835feabac 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonIterator.cs @@ -38,7 +38,7 @@ public PythonIterator(IPythonType iteratorType, IPythonCollection collection) : private IArgumentSet GetArgSet(int index) { var newArg = new PythonConstant(index, Type.DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Int)); - return new ArgumentSet(new List() { newArg }, null, null); + return new ArgumentSet(new List { newArg }, null, null); } public override IMember Call(string memberName, IArgumentSet args) { diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonTypeIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonTypeIterator.cs index 05bf9fa76..e901a292e 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonTypeIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonTypeIterator.cs @@ -20,7 +20,7 @@ namespace Microsoft.Python.Analysis.Values.Collections { internal sealed class PythonTypeIterator : PythonInstance, IPythonIterator { private readonly BuiltinTypeId _contentType; public PythonTypeIterator(BuiltinTypeId iteratorType, BuiltinTypeId contentType, IPythonInterpreter interpreter) - : base(new PythonIteratorType(iteratorType, interpreter)) { + : base(new PythonIteratorType(iteratorType, interpreter.ModuleResolution.BuiltinsModule)) { _contentType = contentType; } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index b44c85045..89d1fa046 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -63,7 +63,6 @@ protected Task CreateServicesAsync(InterpreterConfiguration con protected async Task CreateServicesAsync(string root, InterpreterConfiguration configuration, string stubCacheFolderPath = null, IServiceManager sm = null, string[] searchPaths = null) { configuration = configuration ?? PythonVersions.LatestAvailable; configuration.AssertInstalled(); - stubCacheFolderPath = stubCacheFolderPath ?? TestData.GetAstAnalysisCachePath(configuration.Version, true); Trace.TraceInformation("Cache Path: " + stubCacheFolderPath); searchPaths = searchPaths ?? new[] { GetAnalysisTestDataFilesPath() }; @@ -147,16 +146,16 @@ protected async Task GetAnalysisAsync( doc = new PythonModule(mco, services); } - TestLogger.Log(TraceEventType.Information, "Ast begin"); + TestLogger.Log(TraceEventType.Information, "Test: AST begin."); var ast = await doc.GetAstAsync(CancellationToken.None); ast.Should().NotBeNull(); - TestLogger.Log(TraceEventType.Information, "Ast end"); + TestLogger.Log(TraceEventType.Information, "Test: AST end."); - TestLogger.Log(TraceEventType.Information, "Analysis begin"); + TestLogger.Log(TraceEventType.Information, "Test: Analysis begin."); await services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(-1); analysis.Should().NotBeNull(); - TestLogger.Log(TraceEventType.Information, "Analysis end"); + TestLogger.Log(TraceEventType.Information, "Test: Analysis end."); return analysis; } diff --git a/src/Analysis/Ast/Test/BasicTests.cs b/src/Analysis/Ast/Test/BasicTests.cs index e82c400d1..a469fdddc 100644 --- a/src/Analysis/Ast/Test/BasicTests.cs +++ b/src/Analysis/Ast/Test/BasicTests.cs @@ -13,15 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; -using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -82,56 +79,5 @@ import sys .HaveVariable("sys").OfType(BuiltinTypeId.Module) .And.HaveVariable("x").OfType(BuiltinTypeId.List); } - - [DataRow(true, true)] - [DataRow(false, true)] - [DataRow(true, false)] - [DataRow(false, false)] - [DataTestMethod, Priority(0)] - public async Task UnknownType(bool isPython3X, bool isAnaconda) { - const string code = @"x = 1"; - - var configuration = isPython3X - ? isAnaconda ? PythonVersions.LatestAnaconda3X : PythonVersions.LatestAvailable3X - : isAnaconda ? PythonVersions.LatestAnaconda2X : PythonVersions.LatestAvailable2X; - var analysis = await GetAnalysisAsync(code, configuration); - - var unkType = analysis.Document.Interpreter.UnknownType; - unkType.TypeId.Should().Be(BuiltinTypeId.Unknown); - } - - [DataRow(true, true)] - [DataRow(false, true)] - [DataRow(true, false)] - [DataRow(false, false)] - [DataTestMethod, Priority(0)] - public async Task BuiltinsTest(bool isPython3X, bool isAnaconda) { - const string code = @" -x = 1 -"; - var configuration = isPython3X - ? isAnaconda ? PythonVersions.LatestAnaconda3X : PythonVersions.LatestAvailable3X - : isAnaconda ? PythonVersions.LatestAnaconda2X : PythonVersions.LatestAvailable2X; - var analysis = await GetAnalysisAsync(code, configuration); - - var v = analysis.Should().HaveVariable("x").Which; - var t = v.Value.GetPythonType(); - t.Should().BeAssignableTo(); - - var mc = (IMemberContainer)t; - var names = mc.GetMemberNames().ToArray(); - names.Length.Should().BeGreaterThan(50); - } - - [TestMethod, Priority(0)] - public async Task BuiltinsTrueFalse() { - const string code = @" -booltypetrue = True -booltypefalse = False -"; - var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable(@"booltypetrue").OfType(BuiltinTypeId.Bool) - .And.HaveVariable(@"booltypefalse").OfType(BuiltinTypeId.Bool); - } } } diff --git a/src/Analysis/Ast/Test/BuiltinsTests.cs b/src/Analysis/Ast/Test/BuiltinsTests.cs new file mode 100644 index 000000000..04860f26c --- /dev/null +++ b/src/Analysis/Ast/Test/BuiltinsTests.cs @@ -0,0 +1,99 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class BuiltinsTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + + [DataRow(true, true)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(false, false)] + [DataTestMethod, Priority(0)] + public async Task BuiltinsTest(bool isPython3X, bool isAnaconda) { + const string code = @" +x = 1 +"; + var configuration = isPython3X + ? isAnaconda ? PythonVersions.LatestAnaconda3X : PythonVersions.LatestAvailable3X + : isAnaconda ? PythonVersions.LatestAnaconda2X : PythonVersions.LatestAvailable2X; + var analysis = await GetAnalysisAsync(code, configuration); + + var v = analysis.Should().HaveVariable("x").Which; + var t = v.Value.GetPythonType(); + t.Should().BeAssignableTo(); + + var mc = (IMemberContainer)t; + var names = mc.GetMemberNames().ToArray(); + names.Length.Should().BeGreaterThan(50); + } + + [TestMethod, Priority(0)] + public async Task BuiltinsTrueFalse() { + const string code = @" +booltypetrue = True +booltypefalse = False +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable(@"booltypetrue").OfType(BuiltinTypeId.Bool) + .And.HaveVariable(@"booltypefalse").OfType(BuiltinTypeId.Bool); + } + + [DataRow(true, true)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(false, false)] + [DataTestMethod, Priority(0)] + public async Task UnknownType(bool isPython3X, bool isAnaconda) { + const string code = @"x = 1"; + + var configuration = isPython3X + ? isAnaconda ? PythonVersions.LatestAnaconda3X : PythonVersions.LatestAvailable3X + : isAnaconda ? PythonVersions.LatestAnaconda2X : PythonVersions.LatestAvailable2X; + var analysis = await GetAnalysisAsync(code, configuration); + + var unkType = analysis.Document.Interpreter.UnknownType; + unkType.TypeId.Should().Be(BuiltinTypeId.Unknown); + } + + [TestMethod, Priority(0)] + public async Task Type() { + const string code = @" +class _C: + def _m(self): pass +MethodType = type(_C()._m)"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable(@"MethodType").OfType(BuiltinTypeId.Method); + } + } +} diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 0b1d91c0c..61e3f6f94 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -574,6 +574,27 @@ class C(B): analysis.Should().HaveClass("C").Which.Should().HaveDocumentation("class C doc"); } + [TestMethod, Priority(0)] + public async Task NoDocFromObject() { + const string code = @" +class A(object): ... +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveClass("A").Which.Documentation.Should().BeNull(); + } + + [TestMethod, Priority(0)] + public async Task NoDocFromGeneric() { + const string code = @" +from typing import Generic, TypeVar + +T = TypeVar('T') +class A(Generic[T]): ... +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveClass("A").Which.Documentation.Should().BeNull(); + } + [TestMethod, Priority(0)] public async Task GetAttr() { const string code = @" @@ -635,6 +656,40 @@ def __init__(self): sw.ElapsedMilliseconds.Should().BeLessThan(60000); } + [TestMethod, Priority(0)] + public async Task NestedClasses() { + const string code = @" +class A: + class B: + def __init__(self): ... + + def __init__(self): ... + +x = A.B +ab = A.B() +"; + var analysis = await GetAnalysisAsync(code); + + analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Type); + var ab = analysis.Should().HaveVariable("ab").Which; + ab.Value.Should().BeAssignableTo() + .And.HaveType(typeof(IPythonClassType)); + + var c = ab.Value.GetPythonType(); + c.Should().NotBeNull(); + c.DeclaringType.Name.Should().Be("A"); + c.DeclaringType.Should().BeAssignableTo(); + } + + [TestMethod, Priority(0)] + public async Task MethodType() { + const string code = @" +x = type(object.__init__) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Method); + } + [TestMethod, Priority(0)] public async Task NestedProperty() { const string code = @" diff --git a/src/Analysis/Ast/Test/ConditionalsTests.cs b/src/Analysis/Ast/Test/ConditionalsTests.cs index 054142dd1..a94654a82 100644 --- a/src/Analysis/Ast/Test/ConditionalsTests.cs +++ b/src/Analysis/Ast/Test/ConditionalsTests.cs @@ -168,6 +168,38 @@ def func(a, b): ... .Which.Should().HaveParameters(is3x ? new[] { "a" } : new[] { "a", "b" }); } + [DataRow(false)] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task BigLittleEndian(bool isLittleEndian) { + const string code = @" +if sys.byteorder == 'little': + x = 1 +else: + x = 'a' +"; + var platform = SubstitutePlatform(out var sm); + platform.IsLittleEndian.Returns(x => isLittleEndian); + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X, sm); + analysis.Should().HaveVariable("x").OfType(isLittleEndian ? BuiltinTypeId.Int : BuiltinTypeId.Str); + } + + [DataRow(false)] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task BigLittleEndianNotEqual(bool isLittleEndian) { + const string code = @" +if sys.byteorder != 'little': + x = 1 +else: + x = 'a' +"; + var platform = SubstitutePlatform(out var sm); + platform.IsLittleEndian.Returns(x => isLittleEndian); + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X, sm); + analysis.Should().HaveVariable("x").OfType(!isLittleEndian ? BuiltinTypeId.Int : BuiltinTypeId.Str); + } + private IOSPlatform SubstitutePlatform(out IServiceManager sm) { sm = new ServiceManager(); var platform = Substitute.For(); diff --git a/src/Analysis/Ast/Test/DocumentBufferTests.cs b/src/Analysis/Ast/Test/DocumentBufferTests.cs index 80b8d2b50..c0230ac18 100644 --- a/src/Analysis/Ast/Test/DocumentBufferTests.cs +++ b/src/Analysis/Ast/Test/DocumentBufferTests.cs @@ -234,7 +234,7 @@ public void InsertTopToBottom() { public void NewLines(string s, NewLineLocation[] expected) { var doc = new DocumentBuffer(); doc.Reset(0, s); - var nls = doc.GetNewLineLications().ToArray(); + var nls = doc.GetNewLineLocations().ToArray(); for (var i = 0; i < nls.Length; i++) { Assert.AreEqual(nls[i].Kind, expected[i].Kind); Assert.AreEqual(nls[i].EndIndex, expected[i].EndIndex); diff --git a/src/Analysis/Ast/Test/EnumTests.cs b/src/Analysis/Ast/Test/EnumTests.cs new file mode 100644 index 000000000..d2c0af494 --- /dev/null +++ b/src/Analysis/Ast/Test/EnumTests.cs @@ -0,0 +1,48 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class EnumTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + + [TestMethod, Priority(0)] + public async Task BasicEnum() { + const string code = @" +import enum +x = enum.Enum('a', 'b', 'c') +"; + var analysis = await GetAnalysisAsync(code); + var x = analysis.Should().HaveVariable("x").Which; + } + } +} diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 5d8f4755f..871e0a37a 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -15,12 +15,15 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using FluentAssertions; using FluentAssertions.Execution; using FluentAssertions.Primitives; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; namespace Microsoft.Python.Analysis.Tests.FluentAssertions { @@ -86,7 +89,7 @@ public AndWhichConstraint HaveMember(string NotBeNull(); var t = Subject.GetPythonType(); - var mc = (IMemberContainer) t; + var mc = (IMemberContainer)t; Execute.Assertion.ForCondition(mc != null) .BecauseOf(because, reasonArgs) .FailWith($"Expected {GetName(t)} to be a member container{{reason}}."); @@ -104,9 +107,91 @@ public AndWhichConstraint HaveMember(string return new AndWhichConstraint(this, typedMember); } - public AndConstraint HaveSameMembersAs(IMember m) { - m.Should().BeAssignableTo(); - return HaveMembers(((IMemberContainer)m).GetMemberNames(), string.Empty); + public AndConstraint HaveSameMemberNamesAs(IMember member) { + member.Should().BeAssignableTo(); + return HaveMembers(((IMemberContainer)member).GetMemberNames(), string.Empty); + } + + public void HaveSameMembersAs(IMember other) { + other.Should().BeAssignableTo(); + var otherContainer = (IMemberContainer)other; + + var subjectType = Subject.GetPythonType(); + var subjectMemberNames = subjectType.GetMemberNames().ToArray(); + var otherMemberNames = otherContainer.GetMemberNames().ToArray(); + + var missingNames = otherMemberNames.Except(subjectMemberNames).ToArray(); + var extraNames = subjectMemberNames.Except(otherMemberNames).ToArray(); + + Debug.Assert(missingNames.Length == 0); + missingNames.Should().BeEmpty("Subject has missing names: ", missingNames); + + Debug.Assert(extraNames.Length == 0); + extraNames.Should().BeEmpty("Subject has extra names: ", extraNames); + + foreach (var n in subjectMemberNames.Except(Enumerable.Repeat("__base__", 1))) { + var subjectMember = subjectType.GetMember(n); + var otherMember = otherContainer.GetMember(n); + var subjectMemberType = subjectMember.GetPythonType(); + var otherMemberType = otherMember.GetPythonType(); + + // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. + if (subjectMember is IPythonInstance) { + otherMember.Should().BeAssignableTo(); + } + + subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType, $"Type name: {subjectMemberType.Name}"); + //Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); + + if (subjectMemberType is IPythonClassType subjectClass) { + var otherClass = otherMemberType as IPythonClassType; + otherClass.Should().NotBeNull(); + + if(subjectClass is IGenericType gt) { + otherClass.Should().BeAssignableTo(); + otherClass.IsGeneric.Should().Be(gt.IsGeneric, $"Class name: {subjectClass.Name}"); + } + + // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. + //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); + //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); + } + + if (string.IsNullOrEmpty(subjectMemberType.Documentation)) { + otherMemberType.Documentation.Should().BeNullOrEmpty($"Type name: {subjectMemberType.Name}."); + } else { + Debug.Assert(subjectMemberType.Documentation == otherMemberType.Documentation); + subjectMemberType.Documentation.Should().Be(otherMemberType.Documentation, $"Type name: {subjectMemberType.Name}."); + } + + switch (subjectMemberType.MemberType) { + case PythonMemberType.Class: + // Restored collections (like instance of tuple) turn into classes + // rather than into collections with content since we don't track + // collection content in libraries. We don't compare qualified names + // since original module may be source or a stub and that is not + // preserved during restore. + // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); + break; + case PythonMemberType.Function: + case PythonMemberType.Method: + subjectMemberType.Should().BeAssignableTo(); + otherMemberType.Should().BeAssignableTo(); + if (subjectMemberType is IPythonFunctionType subjectFunction) { + var otherFunction = (IPythonFunctionType)otherMemberType; + subjectFunction.Should().HaveSameOverloadsAs(otherFunction); + } + + break; + case PythonMemberType.Property: + subjectMemberType.Should().BeAssignableTo(); + otherMemberType.Should().BeAssignableTo(); + break; + case PythonMemberType.Unknown: + subjectMemberType.IsUnknown().Should().BeTrue(); + break; + } + } } public AndConstraint HaveMembers(params string[] memberNames) diff --git a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionAssertions.cs index cd1d846dd..f67644743 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionAssertions.cs @@ -13,12 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Linq; using FluentAssertions; using FluentAssertions.Execution; using FluentAssertions.Primitives; -using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; @@ -76,6 +74,17 @@ public AndWhichConstraint Hav return new AndWhichConstraint(this, overloads[index]); } + public void HaveSameOverloadsAs(IPythonFunctionType other, string because = "", params object[] reasonArgs) { + var overloads = Subject.Overloads.ToArray(); + Subject.Should().HaveOverloadCount(other.Overloads.Count); + + for (var i = 0; i < Subject.Overloads.Count; i++) { + var subjectOverload = Subject.Overloads[i]; + var otherOverload = other.Overloads[i]; + subjectOverload.Should().HaveSameParametersAs(otherOverload); + } + } + public AndWhichConstraint HaveParameterAt(int index, string because = "", params object[] reasonArgs) { var overloads = Subject.Overloads.ToArray(); Execute.Assertion.ForCondition(overloads.Length == 1) diff --git a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs index 6ab5b0f8e..e79bcd533 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs @@ -102,6 +102,26 @@ public AndConstraint HaveParameters(IEnumerabl return new AndConstraint(this); } + public AndConstraint HaveSameParametersAs(IPythonFunctionOverload other, string because = "", params object[] reasonArgs) { + var parameters = Subject.Parameters.ToArray(); + var expected = other.Parameters.ToArray(); + + parameters.Should().HaveCount(other.Parameters.Count); + for (var j = 0; j < expected.Length; j++) { + var subjectParam = other.Parameters[j]; + var otherParam = other.Parameters[j]; + subjectParam.Name.Should().Be(otherParam.Name); + + if (subjectParam.Type == null) { + otherParam.Type.Should().BeNull(); + } else { + subjectParam.Type.Name.Should().Be(otherParam.Type.Name); + } + } + + return new AndConstraint(this); + } + public AndConstraint HaveNoParameters(string because = "", params object[] reasonArgs) => HaveParameters(Enumerable.Empty(), because, reasonArgs); diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs index dbc4703bd..f8c4e8928 100644 --- a/src/Analysis/Ast/Test/FunctionTests.cs +++ b/src/Analysis/Ast/Test/FunctionTests.cs @@ -88,8 +88,8 @@ def f(a, b): "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); var pt = analysis.Should().HaveVariable("pt").Which; - pt.Should().HaveType("Point(x, y)").And.HaveMember("x"); - pt.Should().HaveType("Point(x, y)").And.HaveMember("y"); + pt.Should().HaveType("Point").And.HaveMember("x"); + pt.Should().HaveType("Point").And.HaveMember("y"); } [TestMethod, Priority(0)] @@ -567,6 +567,35 @@ def inner(): .And.HaveVariable("z").OfType(BuiltinTypeId.Int); } + [TestMethod, Priority(0)] + public async Task NestedMembers() { + const string code = @" +def outer(): + class innerClass(): ... + def innerFunc(): ... +"; + var analysis = await GetAnalysisAsync(code); + var outer = analysis.Should().HaveFunction("outer").Which as IPythonType; + outer.Should().HaveMember("innerClass"); + outer.Should().HaveMember("innerFunc"); + } + + [TestMethod, Priority(0)] + public async Task NestedPropertyMembers() { + const string code = @" +def outer(): + @property + def p(self): + class innerClass(): ... + def innerFunc(): ... +"; + var analysis = await GetAnalysisAsync(code); + var outer = analysis.Should().HaveFunction("outer").Which as IPythonType; + var p = outer.Should().HaveMember("p").Which as IPythonType; + p.Should().HaveMember("innerClass"); + p.Should().HaveMember("innerFunc"); + } + [TestMethod, Priority(0)] public async Task Deprecated() { const string code = @" diff --git a/src/Analysis/Ast/Test/GenericsTests.cs b/src/Analysis/Ast/Test/GenericsTests.cs index d2b3e52c3..376c6d47e 100644 --- a/src/Analysis/Ast/Test/GenericsTests.cs +++ b/src/Analysis/Ast/Test/GenericsTests.cs @@ -389,6 +389,22 @@ from typing import Dict .And.HaveVariable("y").OfType(BuiltinTypeId.Float); } + [TestMethod, Priority(0)] + [Ignore("https://github.com/microsoft/python-language-server/issues/1454")] + public async Task DictPartialParams() { + const string code = @" +from typing import Dict, Generic, TypeVar + +T = TypeVar('T') +class A(Genetic[T], Dict[T, int]) : ... +"; + + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveClass("A") + .Which.Should().HaveBase("Dict[T, int]"); + } + + [TestMethod, Priority(0)] public async Task GenericDictArg() { const string code = @" @@ -869,6 +885,20 @@ def getT(self) -> T: } [TestMethod, Priority(0)] + public async Task GenericBit() { + const string code = @" +from typing import TypeVar, Generic + +_T = TypeVar('_T') + +class A(Generic[_T]): ... +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Diagnostics.Should().BeEmpty(); + var c = analysis.Should().HaveVariable("A").Which.Value.GetPythonType(); + c.IsGeneric.Should().BeTrue(); + } + public async Task GenericClassBaseChain() { const string code = @" from typing import TypeVar, Generic, List diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 5ce08f316..69fd7bfda 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -223,15 +223,6 @@ public async Task FromFuture() { analysis.Should().HaveFunction("print"); } - [TestMethod, Priority(0)] - public async Task PreferTypeToAny() { - var analysis = await GetAnalysisAsync(@"from TypingConstants import *", PythonVersions.LatestAvailable3X); - analysis.Should().HaveVariable("ONE").Which.Should().HaveType("Any"); - analysis.Should().HaveVariable("TWO").Which.Should().HaveType(BuiltinTypeId.Str); - var a = analysis.Should().HaveClass("A").Which; - a.GetMember("x").Should().HaveType(BuiltinTypeId.Int); - } - [TestMethod, Priority(0)] public async Task StarImportDoesNotOverwriteFunction() { const string code = @" diff --git a/src/Analysis/Ast/Test/LibraryTests.cs b/src/Analysis/Ast/Test/LibraryTests.cs index ddc791936..721fd79d3 100644 --- a/src/Analysis/Ast/Test/LibraryTests.cs +++ b/src/Analysis/Ast/Test/LibraryTests.cs @@ -54,8 +54,12 @@ public async Task Datetime() { module.Name.Should().Be("datetime"); var dt = module.Should().HaveMember("datetime").Which; + dt.DeclaringModule.Name.Should().Be("datetime"); - dt.Should().HaveReadOnlyProperty("day").And.HaveMethod("now") + var day = dt.Should().HaveReadOnlyProperty("day").Which; + day.Documentation.Should().NotBeNullOrEmpty(); + + dt.Should().HaveMethod("now") .Which.Should().BeClassMethod().And.HaveSingleOverload() .Which.Should().HaveReturnType() .Which.Should().HaveMembers( diff --git a/src/Analysis/Ast/Test/LintGenericTests.cs b/src/Analysis/Ast/Test/LintGenericTests.cs index e0ce8b18a..c42161627 100644 --- a/src/Analysis/Ast/Test/LintGenericTests.cs +++ b/src/Analysis/Ast/Test/LintGenericTests.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; -using Microsoft.Python.Parsing.Tests; +using Microsoft.Python.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -37,15 +37,15 @@ public void TestInitialize() [DataRow("x = Generic[str]")] [DataTestMethod, Priority(0)] public async Task GenericNotAllTypeParameters(string decl) { - string code = GenericSetup + decl; + var code = GenericSetup + decl; var analysis = await GetAnalysisAsync(code); analysis.Diagnostics.Should().HaveCount(1); var diagnostic = analysis.Diagnostics.ElementAt(0); - var start = decl.IndexOf("Generic") + 1; + var start = decl.IndexOfOrdinal("Generic") + 1; // adding 1 because SourceSpan.End is exclusive and another 1 because SourceSpan is 1-indexed - var end = decl.IndexOf("]", start) + 2; + var end = decl.IndexOfOrdinal("]", start) + 2; diagnostic.SourceSpan.Should().Be(8, start, 8, end); diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingGenericArguments); @@ -58,14 +58,14 @@ public async Task GenericNotAllTypeParameters(string decl) { [DataRow("x = Generic[T,T]")] [DataTestMethod, Priority(0)] public async Task GenericDuplicateArguments(string decl) { - string code = GenericSetup + decl; + var code = GenericSetup + decl; var analysis = await GetAnalysisAsync(code); analysis.Diagnostics.Should().HaveCount(1); var diagnostic = analysis.Diagnostics.ElementAt(0); - var start = decl.IndexOf("Generic") + 1; + var start = decl.IndexOfOrdinal("Generic") + 1; // adding 1 because SourceSpan.End is exclusive and another 1 because SourceSpan is 1-indexed - var end = decl.IndexOf("]", start) + 2; + var end = decl.IndexOfOrdinal("]", start) + 2; diagnostic.SourceSpan.Should().Be(8, start, 8, end); diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingGenericArguments); @@ -78,14 +78,14 @@ public async Task GenericDuplicateArguments(string decl) { [DataRow("x = Generic[T,T1, _X]")] [DataTestMethod, Priority(0)] public async Task GenericArgumentsNoDiagnosticOnValid(string decl) { - string code = GenericSetup + decl; + var code = GenericSetup + decl; var analysis = await GetAnalysisAsync(code); analysis.Diagnostics.Should().BeEmpty(); } [TestMethod, Priority(0)] public async Task GenericNoArgumentsNoDiagnostic() { - string code = GenericSetup + @" + const string code = GenericSetup + @" x = Generic[] "; var analysis = await GetAnalysisAsync(code); @@ -95,7 +95,7 @@ public async Task GenericNoArgumentsNoDiagnostic() { [TestMethod, Priority(0)] public async Task GenericArgumentSpaceNoDiagnostic() { - string code = GenericSetup + @" + const string code = GenericSetup + @" x = Generic[ ] "; var analysis = await GetAnalysisAsync(code); diff --git a/src/Analysis/Ast/Test/LintNoQATests.cs b/src/Analysis/Ast/Test/LintNoQATests.cs index 172f0bbdc..f1cab156a 100644 --- a/src/Analysis/Ast/Test/LintNoQATests.cs +++ b/src/Analysis/Ast/Test/LintNoQATests.cs @@ -6,7 +6,6 @@ using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Tests.FluentAssertions; -using Microsoft.Python.Core; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -40,7 +39,7 @@ public void TestInitialize() [DataRow("x = Generic[T, T] #noqa ")] [DataTestMethod, Priority(0)] public async Task IgnoreGenerics(string decl) { - string code = GenericSetup + decl; + var code = GenericSetup + decl; var analysis = await GetAnalysisAsync(code); analysis.Diagnostics.Should().BeEmpty(); @@ -81,7 +80,7 @@ public async Task IgnoreNoQAUppercase(string code) { [TestMethod, Priority(0)] public async Task VarNamedNoQAStillGivesDiagnostic() { - string code = GenericSetup + "NOQA = Generic[T, T]"; + const string code = GenericSetup + "NOQA = Generic[T, T]"; var analysis = await GetAnalysisAsync(code); analysis.Diagnostics.Should().HaveCount(1); @@ -103,7 +102,7 @@ public async Task IgnoreUndefinedVar(string code) { [TestMethod, Priority(0)] public async Task IgnoreMissingImport() { - string code = @" + const string code = @" from fake_module import User #noqa "; var analysis = await GetAnalysisAsync(code); diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index 70000c11c..0797b6812 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -42,6 +42,9 @@ + + + diff --git a/src/Analysis/Ast/Test/PathClassificationTests.cs b/src/Analysis/Ast/Test/PathClassificationTests.cs index b6dac71eb..e69de29bb 100644 --- a/src/Analysis/Ast/Test/PathClassificationTests.cs +++ b/src/Analysis/Ast/Test/PathClassificationTests.cs @@ -1,252 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.IO; -using FluentAssertions; -using Microsoft.Python.Analysis.Core.Interpreter; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Core.OS; -using Microsoft.Python.Tests.Utilities.FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using TestUtilities; - -namespace Microsoft.Python.Analysis.Tests { - [TestClass] - public class PathClassificationTests { - private readonly FileSystem _fs = new FileSystem(new OSPlatform()); - - public TestContext TestContext { get; set; } - - [TestInitialize] - public void TestInitialize() - => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - - [TestCleanup] - public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - - [TestMethod] - public void Plain() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var venv = Path.Combine(root, "venv"); - var venvLib = Path.Combine(venv, "Lib"); - var venvSitePackages = Path.Combine(venvLib, "site-packages"); - - var fromInterpreter = new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, Array.Empty()); - - interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - }); - - userPaths.Should().BeEmpty(); - } - - [TestMethod] - public void WithSrcDir() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var venv = Path.Combine(root, "venv"); - var venvLib = Path.Combine(venv, "Lib"); - var venvSitePackages = Path.Combine(venvLib, "site-packages"); - - var src = Path.Combine(root, "src"); - - var fromInterpreter = new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - }; - - var fromUser = new[] { - "./src", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser); - - interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - }); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - }); - } - - [TestMethod] - public void NormalizeUser() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var src = Path.Combine(root, "src"); - - var fromUser = new[] { - "./src/", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); - - interpreterPaths.Should().BeEmpty(); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - }); - } - - [TestMethod] - public void NestedUser() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var src = Path.Combine(root, "src"); - var srcSomething = Path.Combine(src, "something"); - var srcFoo = Path.Combine(src, "foo"); - var srcFooBar = Path.Combine(srcFoo, "bar"); - - var fromUser = new[] { - "./src", - "./src/something", - "./src/foo/bar", - "./src/foo", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); - - interpreterPaths.Should().BeEmpty(); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified), - }); - } - - [TestMethod] - public void NestedUserOrdering() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var src = Path.Combine(root, "src"); - var srcSomething = Path.Combine(src, "something"); - var srcFoo = Path.Combine(src, "foo"); - var srcFooBar = Path.Combine(srcFoo, "bar"); - - var fromUser = new[] { - "./src/foo", - "./src/foo/bar", - "./src", - "./src/something", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); - - interpreterPaths.Should().BeEmpty(); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified), - }); - } - - [TestMethod] - public void InsideStdLib() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var venv = Path.Combine(root, "venv"); - var venvLib = Path.Combine(venv, "Lib"); - var venvSitePackages = Path.Combine(venvLib, "site-packages"); - var inside = Path.Combine(venvSitePackages, "inside"); - var what = Path.Combine(venvSitePackages, "what"); - - var src = Path.Combine(root, "src"); - - var fromInterpreter = new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - new PythonLibraryPath(inside, PythonLibraryPathType.Pth), - }; - - var fromUser = new[] { - "./src", - "./venv/Lib/site-packages/what", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser); - - interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(what, PythonLibraryPathType.Unspecified), - new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), - new PythonLibraryPath(inside, PythonLibraryPathType.Pth), - }); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - }); - } - - [TestMethod] - public void SiteOutsideStdlib() { - var appPath = TestData.GetTestSpecificPath("app.py"); - var root = Path.GetDirectoryName(appPath); - - var venv = Path.Combine(root, "venv"); - var venvLib = Path.Combine(venv, "Lib"); - var sitePackages = Path.Combine(root, "site-packages"); - - var src = Path.Combine(root, "src"); - - var fromInterpreter = new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(sitePackages, PythonLibraryPathType.Site), - }; - - var fromUser = new[] { - "./src", - }; - - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser); - - interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), - new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), - new PythonLibraryPath(sitePackages, PythonLibraryPathType.Site), - }); - - userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { - new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), - }); - } - } -} diff --git a/src/Analysis/Ast/Test/PepHintTests.cs b/src/Analysis/Ast/Test/PepHintTests.cs index 8964ddc7f..1cf706944 100644 --- a/src/Analysis/Ast/Test/PepHintTests.cs +++ b/src/Analysis/Ast/Test/PepHintTests.cs @@ -73,7 +73,7 @@ class Response: # truncated timedelta.IsUnknown().Should().BeFalse(); c.Should().HaveMember("elapsed") - .Which.Should().HaveSameMembersAs(timedelta); + .Which.Should().HaveSameMemberNamesAs(timedelta); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Ast/Test/Properties/AssemblyInfo.cs b/src/Analysis/Ast/Test/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..9d32fae0c --- /dev/null +++ b/src/Analysis/Ast/Test/Properties/AssemblyInfo.cs @@ -0,0 +1,18 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index ceed5b90b..34d6a8caf 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -262,8 +262,7 @@ await FullStdLibTest(v, "win32ui" ); } - - + private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules) { configuration.AssertInstalled(); var moduleUri = TestData.GetDefaultModuleUri(); diff --git a/src/Analysis/Ast/Test/StubMergeTests.cs b/src/Analysis/Ast/Test/StubMergeTests.cs new file mode 100644 index 000000000..1b67e8999 --- /dev/null +++ b/src/Analysis/Ast/Test/StubMergeTests.cs @@ -0,0 +1,66 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class StubMergeTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task Datetime() { + var analysis = await GetAnalysisAsync("import datetime"); + + var module = analysis.Should().HaveVariable("datetime") + .Which.Should().HaveType().Which.Value as IPythonModule; + module.Should().NotBeNull(); + + var stub = module.Stub; + stub.Should().NotBeNull(); + + var dt = stub.Should().HaveClass("datetime").Which; + dt.DeclaringModule.Name.Should().Be("datetime"); + } + + [TestMethod, Priority(0)] + public async Task Os() { + var analysis = await GetAnalysisAsync("import os"); + + var module = analysis.Should().HaveVariable("os") + .Which.Should().HaveType().Which.Value as IPythonModule; + module.Should().NotBeNull(); + + var environ = module.Should().HaveMember("environ").Which; + var environType = environ.GetPythonType(); + environType.Documentation.Should().NotBeNullOrEmpty(); + } + } +} diff --git a/src/Analysis/Ast/Test/TypeshedTests.cs b/src/Analysis/Ast/Test/TypeshedTests.cs index 9b1a72c6f..9ab18657b 100644 --- a/src/Analysis/Ast/Test/TypeshedTests.cs +++ b/src/Analysis/Ast/Test/TypeshedTests.cs @@ -45,7 +45,7 @@ import sys e1, e2, e3 = sys.exc_info() "; var analysis = await GetAnalysisAsync(code); - // sys.exc_info() -> (exception_type, exception_value, traceback) + // def exc_info() -> Tuple[Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]]: ... var f = analysis.Should() .HaveVariable("e1").OfType(BuiltinTypeId.Type) .And.HaveVariable("e2").OfType("BaseException") diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs index f0856a743..31d9fd1de 100644 --- a/src/Analysis/Ast/Test/TypingTests.cs +++ b/src/Analysis/Ast/Test/TypingTests.cs @@ -83,10 +83,25 @@ from typing import TypeVar [TestMethod, Priority(0)] - public async Task KeywordArgMixDocCheck() { + public async Task TypeVarBoundToUnknown() { const string code = @" from typing import TypeVar X = TypeVar('X', bound='hello', covariant=True) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("X") + .Which.Value.Should().HaveDocumentation("TypeVar('X', bound=Unknown, covariant=True)"); + analysis.Should().HaveGenericVariable("X"); + } + + [TestMethod, Priority(0)] + public async Task TypeVarBoundToStringName() { + const string code = @" +from typing import TypeVar + +X = TypeVar('X', bound='hello', covariant=True) + +class hello: ... "; var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("X") @@ -144,7 +159,6 @@ public async Task Containers() { const string code = @" from typing import * -i : SupportsInt = ... lst_i : List[int] = ... lst_i_0 = lst_i[0] @@ -169,8 +183,7 @@ from typing import * ; var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable("i").OfType(BuiltinTypeId.Int) - .And.HaveVariable("lst_i").OfType("List[int]") + analysis.Should().HaveVariable("lst_i").OfType("List[int]") .And.HaveVariable("lst_i_0").OfType(BuiltinTypeId.Int) .And.HaveVariable("u").OfType("Union[Mapping[int, str], MappingView[str, float], MutableMapping[int, List[str]]]") .And.HaveVariable("dct_s_i").OfType("Mapping[str, int]") @@ -235,9 +248,10 @@ from typing import * "; var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable("n1").OfType("n1(x: int, y: float)") + analysis.Should().HaveVariable("n1").OfType("n1") + .Which.Value.Should().HaveDocumentation("n1(x: int, y: float)"); - .And.HaveVariable("n1_x").OfType(BuiltinTypeId.Int) + analysis.Should().HaveVariable("n1_x").OfType(BuiltinTypeId.Int) .And.HaveVariable("n1_y").OfType(BuiltinTypeId.Float) .And.HaveVariable("n1_0").OfType(BuiltinTypeId.Int) @@ -265,9 +279,9 @@ from typing import AnyStr y = n2[0] "; var analysis = await GetAnalysisAsync(code); - analysis.Should().HaveVariable("n1").OfType("AnyStr") - .And.HaveVariable("x").OfType("AnyStr") - .And.HaveVariable("y").OfType("AnyStr"); + analysis.Should().HaveVariable("n1").OfType(BuiltinTypeId.Str) + .And.HaveVariable("x").OfType(BuiltinTypeId.Str) + .And.HaveVariable("y").OfType(BuiltinTypeId.Str); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs index 132d06ff1..57c93cc1d 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Linq; using Microsoft.Python.Parsing.Ast; @@ -20,9 +21,13 @@ namespace Microsoft.Python.Analysis.Core.DependencyResolution { public static class AstUtilities { public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) { var rootNames = fromImportStatement.Root.Names.Select(n => n.Name); - return fromImportStatement.Root is RelativeModuleName relativeName - ? pathResolver.GetImportsFromRelativePath(modulePath, relativeName.DotCount, rootNames) - : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, fromImportStatement.ForceAbsolute); + var dotCount = fromImportStatement.Root is RelativeModuleName relativeName ? relativeName.DotCount : 0; + return pathResolver.FindImports(modulePath, rootNames, dotCount, fromImportStatement.ForceAbsolute); } + + public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, IEnumerable rootNames, int dotCount, bool forceAbsolute) + => dotCount > 0 + ? pathResolver.GetImportsFromRelativePath(modulePath, dotCount, rootNames) + : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, forceAbsolute); } } diff --git a/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs b/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs index 32a88da99..503eb6c6a 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ImportRoot.cs @@ -1,5 +1,4 @@ - -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use diff --git a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs index b229ab80e..9d95188e3 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs @@ -27,6 +27,7 @@ public class ModuleImport : IImportChildrenSource { public bool IsCompiled { get; } public bool IsLibrary { get; } public bool IsBuiltin => IsCompiled && ModulePath == null; + public bool IsPersistent { get; set; } public ModuleImport(IImportChildrenSource childrenSource, string name, string fullName, string rootPath, string modulePath, long moduleFileSize, bool isCompiled, bool isLibrary) { _childrenSource = childrenSource; diff --git a/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs b/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs index 22b115c8b..b500f1e9d 100644 --- a/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs +++ b/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs @@ -15,21 +15,17 @@ using System; using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Core.Interpreter { public sealed class InterpreterConfiguration : IEquatable { - private readonly string _description; - private string _fullDescription; - /// /// Constructs a new interpreter configuration based on the provided values. /// - public InterpreterConfiguration( - string id, - string description, + public InterpreterConfiguration(string interpreterPath = null, Version version = null) : + this(interpreterPath, string.Empty, null, null, default, version) { } + + // Tests only + internal InterpreterConfiguration( string interpreterPath = null, string pathVar = "", string libPath = null, @@ -37,8 +33,6 @@ public InterpreterConfiguration( InterpreterArchitecture architecture = default, Version version = null ) { - Id = id; - _description = description ?? string.Empty; InterpreterPath = interpreterPath; PathEnvironmentVariable = pathVar; Architecture = architecture ?? InterpreterArchitecture.Unknown; @@ -51,8 +45,6 @@ private static string Read(IReadOnlyDictionary d, string k) => d.TryGetValue(k, out var o) ? o as string : null; private InterpreterConfiguration(IReadOnlyDictionary properties) { - Id = Read(properties, nameof(Id)); - _description = Read(properties, nameof(Description)) ?? ""; InterpreterPath = Read(properties, nameof(InterpreterPath)); PathEnvironmentVariable = Read(properties, nameof(PathEnvironmentVariable)); LibraryPath = Read(properties, nameof(LibraryPath)); @@ -65,8 +57,6 @@ private InterpreterConfiguration(IReadOnlyDictionary properties) } public void WriteToDictionary(IDictionary properties) { - properties[nameof(Id)] = Id; - properties[nameof(Description)] = _description; properties[nameof(InterpreterPath)] = InterpreterPath; properties[nameof(PathEnvironmentVariable)] = PathEnvironmentVariable; properties[nameof(LibraryPath)] = LibraryPath; @@ -76,52 +66,6 @@ public void WriteToDictionary(IDictionary properties) { } } - /// - /// Reconstructs an interpreter configuration from a dictionary. - /// - public static InterpreterConfiguration FromDictionary(IReadOnlyDictionary properties) - => new InterpreterConfiguration(properties); - - /// - /// Serializes an interpreter configuration to a dictionary. - /// - public IReadOnlyDictionary ToDictionary() { - var d = new Dictionary(); - WriteToDictionary(d); - return d; - } - - /// - /// Gets a unique and stable identifier for this interpreter. - /// - public string Id { get; } - - /// - /// Gets a friendly description of the interpreter - /// - public string Description => _fullDescription ?? _description; - - /// - /// Changes the description to be less likely to be - /// ambiguous with other interpreters. - /// - public void SwitchToFullDescription() { - var hasVersion = _description.Contains(Version.ToString()); - var hasArch = _description.IndexOf(Architecture.ToString(null, CultureInfo.CurrentCulture), StringComparison.CurrentCultureIgnoreCase) >= 0 || - _description.IndexOf(Architecture.ToString("x", CultureInfo.CurrentCulture), StringComparison.CurrentCultureIgnoreCase) >= 0; - - if (hasVersion && hasArch) { - // Regular description is sufficient - _fullDescription = null; - } else if (hasVersion) { - _fullDescription = "{0} ({1})".FormatUI(_description, Architecture); - } else if (hasArch) { - _fullDescription = "{0} ({1})".FormatUI(_description, Version); - } else { - _fullDescription = "{0} ({1}, {2})".FormatUI(_description, Version, Architecture); - } - } - /// /// Returns the path to the interpreter executable for launching Python /// applications. @@ -166,8 +110,7 @@ public bool Equals(InterpreterConfiguration other) { } var cmp = StringComparer.OrdinalIgnoreCase; - return string.Equals(Id, other.Id) && - cmp.Equals(Description, other.Description) && + return cmp.Equals(InterpreterPath, other.InterpreterPath) && cmp.Equals(PathEnvironmentVariable, other.PathEnvironmentVariable) && Architecture == other.Architecture && @@ -176,28 +119,13 @@ public bool Equals(InterpreterConfiguration other) { public override int GetHashCode() { var cmp = StringComparer.OrdinalIgnoreCase; - return Id.GetHashCode() ^ - cmp.GetHashCode(Description) ^ + return cmp.GetHashCode(InterpreterPath ?? "") ^ cmp.GetHashCode(PathEnvironmentVariable ?? "") ^ Architecture.GetHashCode() ^ Version.GetHashCode(); } - public override string ToString() => Description; - - /// - /// Attempts to update descriptions to be unique within the - /// provided sequence by adding information about the - /// interpreter that is missing from the default description. - /// - public static void DisambiguateDescriptions(IReadOnlyList configs) { - foreach (var c in configs) { - c._fullDescription = null; - } - foreach (var c in configs.GroupBy(i => i._description ?? "").Where(g => g.Count() > 1).SelectMany(g => g)) { - c.SwitchToFullDescription(); - } - } + public override string ToString() => InterpreterPath; } } diff --git a/src/Analysis/Core/Impl/Interpreter/LibraryType.cs b/src/Analysis/Core/Impl/Interpreter/LibraryType.cs new file mode 100644 index 000000000..851ec7d29 --- /dev/null +++ b/src/Analysis/Core/Impl/Interpreter/LibraryType.cs @@ -0,0 +1,38 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Core.Interpreter { + public enum LibraryType { + /// + /// Module is not a library. + /// + None, + + /// + /// Library is part of the standard Python installation. + /// + Standard, + + /// + /// Library is installed in site-packages. + /// + SitePackages, + + /// + /// Other type of library. + /// + Other + } +} diff --git a/src/Analysis/Core/Impl/Interpreter/ModulePath.cs b/src/Analysis/Core/Impl/Interpreter/ModulePath.cs index 99dba57f1..206b01f21 100644 --- a/src/Analysis/Core/Impl/Interpreter/ModulePath.cs +++ b/src/Analysis/Core/Impl/Interpreter/ModulePath.cs @@ -15,11 +15,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; using System.Text.RegularExpressions; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; diff --git a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs index 48a9f79c2..671845dd0 100644 --- a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs +++ b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs @@ -41,16 +41,9 @@ public PythonLibraryPath(string path, PythonLibraryPathType type = PythonLibrary ModulePrefix = modulePrefix ?? string.Empty; } - public PythonLibraryPath(string path, bool isStandardLibrary, string modulePrefix) : - this(path, isStandardLibrary ? PythonLibraryPathType.StdLib : PythonLibraryPathType.Unspecified, modulePrefix) { } - public string Path { get; } - public PythonLibraryPathType Type { get; } - - public string ModulePrefix { get; } = string.Empty; - - public bool IsStandardLibrary => Type == PythonLibraryPathType.StdLib; + public string ModulePrefix { get; } public override string ToString() { var type = string.Empty; @@ -70,9 +63,9 @@ public override string ToString() { return "{0}|{1}|{2}".FormatInvariant(Path, type, ModulePrefix); } - public static PythonLibraryPath Parse(string s) { + private static PythonLibraryPath Parse(string s) { if (string.IsNullOrEmpty(s)) { - throw new ArgumentNullException("source"); + throw new ArgumentNullException(nameof(s)); } var parts = s.Split(new[] { '|' }, 3); @@ -84,8 +77,7 @@ public static PythonLibraryPath Parse(string s) { var ty = parts[1]; var prefix = parts[2]; - PythonLibraryPathType type = PythonLibraryPathType.Unspecified; - + var type = PythonLibraryPathType.Unspecified; switch (ty) { case "stdlib": type = PythonLibraryPathType.StdLib; @@ -108,7 +100,6 @@ public static PythonLibraryPath Parse(string s) { /// File system /// Root of the standard library. /// A list of search paths for the interpreter. - /// New in 2.2, moved in 3.3 private static ImmutableArray GetDefaultSearchPaths(IFileSystem fs, string library) { var result = ImmutableArray.Empty; if (!Directory.Exists(library)) { @@ -165,19 +156,15 @@ public static async Task> GetSearchPathsAsync( public static async Task> GetSearchPathsFromInterpreterAsync(string interpreter, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { // sys.path will include the working directory, so we make an empty // path that we can filter out later - var tempWorkingDir = IOPath.Combine(IOPath.GetTempPath(), IOPath.GetRandomFileName()); - fs.CreateDirectory(tempWorkingDir); - if (!InstallPath.TryGetFile("get_search_paths.py", out var srcGetSearchPaths)) { - return ImmutableArray.Empty; + if (!InstallPath.TryGetFile("get_search_paths.py", out var getSearchPathScript)) { + return new ImmutableArray(); } - var getSearchPaths = IOPath.Combine(tempWorkingDir, PathUtils.GetFileName(srcGetSearchPaths)); - File.Copy(srcGetSearchPaths, getSearchPaths); var startInfo = new ProcessStartInfo( interpreter, - new[] { "-S", "-E", getSearchPaths }.AsQuotedArguments() + new[] { "-S", "-E", getSearchPathScript }.AsQuotedArguments() ) { - WorkingDirectory = tempWorkingDir, + WorkingDirectory = IOPath.GetDirectoryName(getSearchPathScript), UseShellExecute = false, ErrorDialog = false, CreateNoWindow = true, @@ -185,17 +172,11 @@ public static async Task> GetSearchPathsFromIn RedirectStandardOutput = true }; - try { - var output = await ps.ExecuteAndCaptureOutputAsync(startInfo, cancellationToken); - return output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).Select(s => { + var output = await ps.ExecuteAndCaptureOutputAsync(startInfo, cancellationToken); + return output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) + .Skip(1).Select(s => { try { - var p = Parse(s); - - if (PathUtils.PathStartsWith(p.Path, tempWorkingDir)) { - return null; - } - - return p; + return Parse(s); } catch (ArgumentException) { Debug.Fail("Invalid search path: " + (s ?? "")); return null; @@ -204,17 +185,14 @@ public static async Task> GetSearchPathsFromIn return null; } }).Where(p => p != null).ToImmutableArray(); - } finally { - fs.DeleteDirectory(tempWorkingDir, true); - } } public static (ImmutableArray interpreterPaths, ImmutableArray userPaths) ClassifyPaths( - string root, - IFileSystem fs, - IEnumerable fromInterpreter, - IEnumerable fromUser - ) { + string root, + IFileSystem fs, + IEnumerable fromInterpreter, + IEnumerable fromUser + ) { #if DEBUG Debug.Assert(root == null || root.PathEquals(PathUtils.NormalizePathAndTrim(root))); Debug.Assert(!fromInterpreter.Any(p => !p.Path.PathEquals(PathUtils.NormalizePathAndTrim(p.Path)))); @@ -294,7 +272,6 @@ public bool Equals(PythonLibraryPath other) { } public static bool operator ==(PythonLibraryPath left, PythonLibraryPath right) => left?.Equals(right) ?? right is null; - public static bool operator !=(PythonLibraryPath left, PythonLibraryPath right) => !(left?.Equals(right) ?? right is null); } } diff --git a/src/Analysis/Core/Impl/Properties/AssemblyInfo.cs b/src/Analysis/Core/Impl/Properties/AssemblyInfo.cs index 33b24b71a..cd223fd12 100644 --- a/src/Analysis/Core/Impl/Properties/AssemblyInfo.cs +++ b/src/Analysis/Core/Impl/Properties/AssemblyInfo.cs @@ -17,4 +17,5 @@ [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.Parsing.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs b/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs new file mode 100644 index 000000000..b200daf5c --- /dev/null +++ b/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs @@ -0,0 +1,41 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Core.Collections; + +namespace Microsoft.Python.Analysis.Caching { + internal static class DependencyCollectorExtensions { + public static void AddImports(this DependencyCollector dc, IEnumerable imports) { + foreach (var imp in imports) { + foreach (var dottedName in imp.ModuleNames) { + var importNames = ImmutableArray.Empty; + foreach (var part in dottedName.NameParts) { + importNames = importNames.Add(part); + dc.AddImport(importNames, imp.ForceAbsolute); + } + } + } + } + + public static void AddFromImports(this DependencyCollector dc, IEnumerable imports) { + foreach (var imp in imports) { + dc.AddFromImport(imp.RootNames, imp.MemberNames, imp.DotCount, imp.ForceAbsolute); + } + } + } +} diff --git a/src/Caching/Impl/Extensions/IndexSpanExtensions.cs b/src/Caching/Impl/Extensions/IndexSpanExtensions.cs new file mode 100644 index 000000000..3381df3f7 --- /dev/null +++ b/src/Caching/Impl/Extensions/IndexSpanExtensions.cs @@ -0,0 +1,29 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.Analysis.Caching { + internal static class IndexSpanExtensions { + [DebuggerStepThrough] + public static IndexSpanModel ToModel(this IndexSpan span) => new IndexSpanModel { + Start = span.Start, + Length = span.Length + }; + } +} diff --git a/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj b/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj new file mode 100644 index 000000000..73ddbba31 --- /dev/null +++ b/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj @@ -0,0 +1,44 @@ + + + netstandard2.1 + Microsoft.Python.Analysis.Caching + Microsoft.Python.Analysis.Caching + + + + 1701;1702;$(NoWarn) + 7.2 + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs new file mode 100644 index 000000000..323d5646f --- /dev/null +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -0,0 +1,86 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Core; +// ReSharper disable MemberCanBeProtected.Global +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal abstract class CallableModel : MemberModel { + public string Documentation { get; set; } + public FunctionAttributes Attributes { get; set; } + public ClassModel[] Classes { get; set; } + public FunctionModel[] Functions { get; set; } + + [NonSerialized] + private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + protected CallableModel() { } // For de-serializer from JSON + + protected CallableModel(IPythonType callable) { + var functions = new List(); + var classes = new List(); + + foreach (var name in callable.GetMemberNames()) { + var m = callable.GetMember(name); + + // Only take members from this class, skip members from bases. + using (_processing.Push(m, out var reentered)) { + if (reentered) { + continue; + } + switch (m) { + case IPythonFunctionType ft1 when ft1.IsLambda(): + break; + case IPythonFunctionType ft2: + functions.Add(new FunctionModel(ft2)); + break; + case IPythonClassType cls: + classes.Add(new ClassModel(cls)); + break; + } + } + } + + Id = callable.Name.GetStableHash(); + Name = callable.Name; + QualifiedName = callable.QualifiedName; + Documentation = callable.Documentation; + Classes = classes.ToArray(); + Functions = functions.ToArray(); + IndexSpan = callable.Location.IndexSpan.ToModel(); + + Attributes = FunctionAttributes.Normal; + if (callable.IsAbstract) { + Attributes |= FunctionAttributes.Abstract; + } + if(callable is IPythonFunctionType ft) { + if(ft.IsClassMethod) { + Attributes |= FunctionAttributes.ClassMethod; + } + if (ft.IsStatic) { + Attributes |= FunctionAttributes.Static; + } + } + } + + protected override IEnumerable GetMemberModels() => Classes.Concat(Functions); + } +} diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs new file mode 100644 index 000000000..527816157 --- /dev/null +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -0,0 +1,201 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("cls:{" + nameof(Name) + "}")] + internal sealed class ClassModel : MemberModel { + public string Documentation { get; set; } + public string[] Bases { get; set; } + public NamedTupleModel[] NamedTupleBases { get; set; } + public FunctionModel[] Methods { get; set; } + public PropertyModel[] Properties { get; set; } + public VariableModel[] Fields { get; set; } + public ClassModel[] Classes { get; set; } + + /// + /// GenericParameters of the Generic[...] base class, if any. + /// + public string[] GenericBaseParameters { get; set; } + /// + /// Values assigned to the generic parameters, if any. + /// + public GenericParameterValueModel[] GenericParameterValues { get; set; } + + [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + [NonSerialized] private PythonClassType _cls; + + public ClassModel() { } // For de-serializer from JSON + + public ClassModel(IPythonClassType cls) { + var methods = new List(); + var properties = new List(); + var fields = new List(); + var innerClasses = new List(); + + // Skip certain members in order to avoid infinite recursion. + foreach (var name in cls.GetMemberNames().Except(new[] { "__base__", "__bases__", "__class__", "mro" })) { + var m = cls.GetMember(name); + + // Only take members from this class, skip members from bases. + if (m is IPythonClassMember cm && cls.QualifiedName != cm.DeclaringType?.QualifiedName) { + continue; + } + + using (_processing.Push(m, out var reentered)) { + if (reentered) { + continue; + } + + switch (m) { + case IPythonClassType ct when ct.Name == name: + if (!ct.DeclaringModule.Equals(cls.DeclaringModule)) { + continue; + } + innerClasses.Add(new ClassModel(ct)); + break; + case IPythonFunctionType ft when ft.IsLambda(): + break; + case IPythonFunctionType ft when ft.Name == name: + methods.Add(new FunctionModel(ft)); + break; + case IPythonPropertyType prop when prop.Name == name: + properties.Add(new PropertyModel(prop)); + break; + case IPythonInstance inst: + fields.Add(VariableModel.FromInstance(name, inst)); + break; + case IPythonType t: + fields.Add(VariableModel.FromType(name, t)); + break; + } + } + } + + Name = cls.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : cls.Name; + Id = Name.GetStableHash(); + QualifiedName = cls.QualifiedName; + IndexSpan = cls.Location.IndexSpan.ToModel(); + + // Only persist documentation from this class, leave bases or __init__ alone. + Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; + + var ntBases = cls.Bases.OfType().ToArray(); + NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray(); + + Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName()).ToArray(); + Methods = methods.ToArray(); + Properties = properties.ToArray(); + Fields = fields.ToArray(); + Classes = innerClasses.ToArray(); + + if (cls.IsGeneric) { + // Only check immediate bases, i.e. when class itself has Generic[T] base. + var gcp = cls.Bases.OfType().FirstOrDefault(); + GenericBaseParameters = gcp?.TypeParameters.Select(p => p.Name).ToArray(); + } + // If class is generic, we must save its generic base definition + // so on restore we'll be able to re-create the class as generic. + GenericBaseParameters = GenericBaseParameters ?? Array.Empty(); + + GenericParameterValues = cls.GenericParameters + .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName() }) + .ToArray(); + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + if(_cls != null) { + return _cls; + } + _cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan())); + var bases = CreateBases(mf, gs); + + _cls.SetBases(bases); + _cls.SetDocumentation(Documentation); + + if (GenericParameterValues.Length > 0) { + _cls.StoreGenericParameters( + _cls, + _cls.GenericParameters.Keys.ToArray(), + GenericParameterValues.ToDictionary( + k => _cls.GenericParameters.Keys.First(x => x == k.Name), + v => mf.ConstructType(v.Type) + ) + ); + } + + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + _cls.AddMember(m.Name, m.Create(mf, _cls, gs), false); + } + return _cls; + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + m.Populate(mf, _cls, gs); + } + } + + private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { + var ntBases = NamedTupleBases.Select(ntb => { + var n = ntb.Create(mf, _cls, gs); + ntb.Populate(mf, _cls, gs); + return n; + }).OfType().ToArray(); + + var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); + var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); + + if (GenericBaseParameters.Length > 0) { + // Generic class. Need to reconstruct generic base so code can then + // create specific types off the generic class. + var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); + if (genericBase != null) { + var typeVars = GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); + if (typeVars.Length > 0) { + var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); + if (genericWithParameters != null) { + bases = bases.Except(Enumerable.Repeat(genericBase, 1)).Concat(Enumerable.Repeat(genericWithParameters, 1)).ToArray(); + } + } + } else { + Debug.Fail("Generic class does not have generic base."); + } + } + return bases; + } + + protected override IEnumerable GetMemberModels() + => Classes.Concat(Methods).Concat(Properties).Concat(Fields); + } +} diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFile.cs b/src/Caching/Impl/Models/DottedNameModel.cs similarity index 71% rename from src/Analysis/Ast/Impl/Types/Definitions/IPythonFile.cs rename to src/Caching/Impl/Models/DottedNameModel.cs index 897c7bf48..1bbc05822 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFile.cs +++ b/src/Caching/Impl/Models/DottedNameModel.cs @@ -15,16 +15,9 @@ using System; -namespace Microsoft.Python.Analysis.Types { - public interface IPythonFile { - /// - /// File path to the module. - /// - string FilePath { get; } - - /// - /// Module URI. - /// - Uri Uri { get; } +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal sealed class DottedNameModel { + public string[] NameParts { get; set; } } } diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs new file mode 100644 index 000000000..f5a670056 --- /dev/null +++ b/src/Caching/Impl/Models/FromImportModel.cs @@ -0,0 +1,26 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents from import statement for dependency resolution. + /// + internal sealed class FromImportModel { + public string[] RootNames { get; set; } + public string[] MemberNames { get; set; } + public int DotCount { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Analysis/Ast/Impl/Extensions/QualifiedNameExtensions.cs b/src/Caching/Impl/Models/FunctionAttributes.cs similarity index 60% rename from src/Analysis/Ast/Impl/Extensions/QualifiedNameExtensions.cs rename to src/Caching/Impl/Models/FunctionAttributes.cs index 09e7d64a3..1c0f3da74 100644 --- a/src/Analysis/Ast/Impl/Extensions/QualifiedNameExtensions.cs +++ b/src/Caching/Impl/Models/FunctionAttributes.cs @@ -13,15 +13,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; +using System; -namespace Microsoft.Python.Analysis { - internal static class QualifiedNameExtensions { - public static string CombineNames(this KeyValuePair qualifiedNamePair, string sep = ".") { - if (string.IsNullOrEmpty(qualifiedNamePair.Key)) { - return qualifiedNamePair.Value; - } - return qualifiedNamePair.Key + sep + qualifiedNamePair.Value; - } +namespace Microsoft.Python.Analysis.Caching.Models { + [Flags] + internal enum FunctionAttributes { + Normal, + Abstract, + Static, + ClassMethod } } diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs new file mode 100644 index 000000000..dbfd63ddf --- /dev/null +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -0,0 +1,75 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("f:{" + nameof(Name) + "}")] + internal sealed class FunctionModel : CallableModel { + public OverloadModel[] Overloads { get; set; } + public FunctionModel() { } // For de-serializer from JSON + + [NonSerialized] private PythonFunctionType _function; + + public FunctionModel(IPythonFunctionType func) : base(func) { + Overloads = func.Overloads.Select(FromOverload).ToArray(); + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _function ?? (_function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation)); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + // Create inner functions and classes first since function may be returning one of them. + var all = Classes.Concat(Functions).ToArray(); + + foreach (var model in all) { + _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true); + } + foreach (var model in all) { + model.Populate(mf, _function, gs); + } + + foreach (var om in Overloads) { + var o = new PythonFunctionOverload(_function, new Location(mf.Module, IndexSpan.ToSpan())); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray()); + _function.AddOverload(o); + } + } + + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) + => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); + + private static OverloadModel FromOverload(IPythonFunctionOverload o) + => new OverloadModel { + Parameters = o.Parameters.Select(p => new ParameterModel { + Name = p.Name, + Type = p.Type.GetPersistentQualifiedName(), + Kind = p.Kind, + DefaultValue = p.DefaultValue.GetPersistentQualifiedName(), + }).ToArray(), + ReturnType = o.StaticReturnValue.GetPersistentQualifiedName() + }; + } +} diff --git a/src/Caching/Impl/Models/GenericParameterValueModel.cs b/src/Caching/Impl/Models/GenericParameterValueModel.cs new file mode 100644 index 000000000..17f0c68f6 --- /dev/null +++ b/src/Caching/Impl/Models/GenericParameterValueModel.cs @@ -0,0 +1,32 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Model for actual values assigned to generic parameters. + /// I.e. if class is based on Generic[T], what is assigned to T. + /// + internal sealed class GenericParameterValueModel { + /// + /// Generic parameter name as defined by TypeVar, such as T. + /// + public string Name { get; set; } + + /// + /// Qualified name of the type assigned to T. + /// + public string Type { get; set; } + } +} diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs new file mode 100644 index 000000000..42d479291 --- /dev/null +++ b/src/Caching/Impl/Models/ImportModel.cs @@ -0,0 +1,24 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents import statement for dependency resolution. + /// + internal sealed class ImportModel { + public DottedNameModel[] ModuleNames { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Caching/Impl/Models/IndexSpanModel.cs b/src/Caching/Impl/Models/IndexSpanModel.cs new file mode 100644 index 000000000..b9ccc45d7 --- /dev/null +++ b/src/Caching/Impl/Models/IndexSpanModel.cs @@ -0,0 +1,28 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using Microsoft.Python.Core.Text; +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [DebuggerDisplay("{Start}:({Length})")] + internal sealed class IndexSpanModel { + public int Start { get; set; } + public int Length { get; set; } + public IndexSpan ToSpan() => new IndexSpan(Start, Length); + } +} diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs new file mode 100644 index 000000000..e7ce43282 --- /dev/null +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -0,0 +1,61 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +// ReSharper disable MemberCanBeProtected.Global +// ReSharper disable UnusedAutoPropertyAccessor.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal abstract class MemberModel { + /// + /// Member unique id in the database. + /// + public int Id { get; set; } + + /// + /// Member name, such as name of a class. + /// + public string Name { get; set; } + + /// + /// Member qualified name within the module, such as A.B.C. + /// + public string QualifiedName { get; set; } + + /// + /// Member location in the module original source code. + /// + public IndexSpanModel IndexSpan { get; set; } + + /// + /// Create member for declaration but does not construct its parts just yet. + /// Used as a first pass in two-pass handling of forward declarations. + /// + public abstract IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + + /// + /// Populate member with content, such as create class methods, etc. + /// + public abstract void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + + public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); + protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); + } +} diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs new file mode 100644 index 000000000..fbca1705f --- /dev/null +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -0,0 +1,218 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal sealed class ModuleModel : MemberModel { + /// + /// Module unique id that includes version. + /// + public string UniqueId { get; set; } + + public string Documentation { get; set; } + public FunctionModel[] Functions { get; set; } + public VariableModel[] Variables { get; set; } + public ClassModel[] Classes { get; set; } + public TypeVarModel[] TypeVars { get; set; } + public NamedTupleModel[] NamedTuples { get; set; } + + /// + /// Collection of new line information for conversion of linear spans + /// to line/columns in navigation to member definitions and references. + /// + public NewLineModel[] NewLines { get; set; } + + /// + /// Length of the original module file. Used in conversion of indices to line/columns. + /// + public int FileSize { get; set; } + + public ImportModel[] Imports { get; set; } + public FromImportModel[] FromImports { get; set; } + public ImportModel[] StubImports { get; set; } + public FromImportModel[] StubFromImports { get; set; } + + [NonSerialized] private Dictionary _modelCache; + + public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { + var uniqueId = analysis.Document.GetUniqueId(services, options); + if (uniqueId == null) { + // Caching level setting does not permit this module to be persisted. + return null; + } + + var variables = new Dictionary(); + var functions = new Dictionary(); + var classes = new Dictionary(); + var typeVars = new Dictionary(); + var namedTuples = new Dictionary(); + + // Go directly through variables which names are listed in GetMemberNames + // as well as variables that are declarations. + var exportedNames = new HashSet(analysis.Document.GetMemberNames()); + foreach (var v in analysis.GlobalScope.Variables + .Where(v => exportedNames.Contains(v.Name) || + v.Source == VariableSource.Declaration || + v.Source == VariableSource.Builtin || + v.Source == VariableSource.Generic)) { + + if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { + typeVars[v.Name] = TypeVarModel.FromGeneric(v); + continue; + } + + switch (v.Value) { + case ITypingNamedTupleType nt: + namedTuples[nt.Name] = new NamedTupleModel(nt); + continue; + case IPythonFunctionType ft when ft.IsLambda(): + // No need to persist lambdas. + continue; + case IPythonFunctionType ft when v.Name != ft.Name: + // Variable assigned to type info of the function like + // def func(): ... + // x = type(func) + break; + case IPythonFunctionType ft: + var fm = GetFunctionModel(analysis, v, ft); + if (fm != null && !functions.ContainsKey(ft.Name)) { + functions[ft.Name] = fm; + continue; + } + break; + case IPythonClassType cls when v.Name != cls.Name: + // Variable assigned to type info of the class. + break; + case IPythonClassType cls + when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub): + if (!classes.ContainsKey(cls.Name)) { + classes[cls.Name] = new ClassModel(cls); + continue; + } + break; + } + + // Do not re-declare classes and functions as variables in the model. + if (!variables.ContainsKey(v.Name)) { + variables[v.Name] = VariableModel.FromVariable(v); + } + } + + // Take dependencies from imports. If module has stub we should also take + // dependencies from there since persistent state is based on types that + // are combination of stub and the module. Sometimes stub may import more + // and we must make sure dependencies are restored before the module. + var primaryDependencyWalker = new DependencyWalker(analysis.Ast); + var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null; + var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty(); + var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty(); + + return new ModuleModel { + Id = uniqueId.GetStableHash(), + UniqueId = uniqueId, + Name = analysis.Document.Name, + QualifiedName = analysis.Document.QualifiedName, + Documentation = analysis.Document.Documentation, + Functions = functions.Values.ToArray(), + Variables = variables.Values.ToArray(), + Classes = classes.Values.ToArray(), + TypeVars = typeVars.Values.ToArray(), + NamedTuples = namedTuples.Values.ToArray(), + NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel { + EndIndex = l.EndIndex, + Kind = l.Kind + }).ToArray(), + FileSize = analysis.Ast.EndIndex, + Imports = primaryDependencyWalker.Imports.ToArray(), + FromImports = primaryDependencyWalker.FromImports.ToArray(), + StubImports = stubImports.ToArray(), + StubFromImports = stubFromImports.ToArray() + }; + } + + private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVariable v, IPythonFunctionType f) { + if (v.Source == VariableSource.Import && !f.DeclaringModule.Equals(analysis.Document) && !f.DeclaringModule.Equals(analysis.Document.Stub)) { + // It may be that the function is from a child module via import. + // For example, a number of functions in 'os' are imported from 'nt' on Windows via + // star import. Their stubs, however, come from 'os' stub. The function then have declaring + // module as 'nt' rather than 'os' and 'nt' does not have a stub. In this case use function + // model like if function was declared in 'os'. + return new FunctionModel(f); + } + + if (f.DeclaringModule.Equals(analysis.Document) || f.DeclaringModule.Equals(analysis.Document.Stub)) { + return new FunctionModel(f); + } + return null; + } + + public override MemberModel GetModel(string name) { + if (_modelCache == null) { + var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); + _modelCache = new Dictionary(); + foreach (var m in models) { + Debug.Assert(!_modelCache.ContainsKey(m.Name)); + _modelCache[m.Name] = m; + } + } + return _modelCache.TryGetValue(name, out var model) ? model : null; + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + + private sealed class DependencyWalker : PythonWalker { + public List Imports { get; } = new List(); + public List FromImports { get; } = new List(); + + public DependencyWalker(PythonAst ast) { + ast.Walk(this); + } + + public override bool Walk(ImportStatement import) { + var model = new ImportModel { + ForceAbsolute = import.ForceAbsolute, + ModuleNames = import.Names.Select(mn => new DottedNameModel { + NameParts = mn.Names.Select(nex => nex.Name).ToArray() + }).ToArray() + }; + Imports.Add(model); + return false; + } + + public override bool Walk(FromImportStatement fromImport) { + var model = new FromImportModel { + ForceAbsolute = fromImport.ForceAbsolute, + RootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(), + MemberNames = fromImport.Names.Select(n => n.Name).ToArray(), + DotCount = fromImport.Root is RelativeModuleName rn ? rn.DotCount : 0 + }; + FromImports.Add(model); + return false; + } + } + } +} diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs new file mode 100644 index 000000000..82fed074b --- /dev/null +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -0,0 +1,57 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("n:{" + nameof(Name) + "}")] + internal sealed class NamedTupleModel: MemberModel { + public string[] ItemNames { get; set; } + public string[] ItemTypes { get; set; } + + [NonSerialized] private NamedTupleType _namedTuple; + + public NamedTupleModel() { } // For de-serializer from JSON + + public NamedTupleModel(ITypingNamedTupleType nt) { + Id = nt.Name.GetStableHash(); + Name = nt.Name; + QualifiedName = nt.QualifiedName; + IndexSpan = nt.Location.IndexSpan.ToModel(); + ItemNames = nt.ItemNames.ToArray(); + ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + if (_namedTuple != null) { + return _namedTuple; + } + + var itemTypes = ItemTypes.Select(mf.ConstructType).ToArray(); + _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan()); + return _namedTuple; + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + } +} diff --git a/src/Caching/Impl/Models/NewLineModel.cs b/src/Caching/Impl/Models/NewLineModel.cs new file mode 100644 index 000000000..b3021102d --- /dev/null +++ b/src/Caching/Impl/Models/NewLineModel.cs @@ -0,0 +1,24 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.Analysis.Caching.Models { + internal sealed class NewLineModel { + public int EndIndex { get; set; } + public NewLineKind Kind { get; set; } + } +} diff --git a/src/Caching/Impl/Models/OverloadModel.cs b/src/Caching/Impl/Models/OverloadModel.cs new file mode 100644 index 000000000..1f0014eef --- /dev/null +++ b/src/Caching/Impl/Models/OverloadModel.cs @@ -0,0 +1,21 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + internal sealed class OverloadModel { + public ParameterModel[] Parameters { get; set; } + public string ReturnType { get; set; } + } +} diff --git a/src/Caching/Impl/Models/ParameterModel.cs b/src/Caching/Impl/Models/ParameterModel.cs new file mode 100644 index 000000000..6006b385b --- /dev/null +++ b/src/Caching/Impl/Models/ParameterModel.cs @@ -0,0 +1,25 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Models { + internal sealed class ParameterModel { + public string Name { get; set; } + public string Type { get; set; } + public string DefaultValue { get; set; } + public ParameterKind Kind { get; set; } + } +} diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs new file mode 100644 index 000000000..bea252b81 --- /dev/null +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -0,0 +1,47 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal sealed class PropertyModel : CallableModel { + public string ReturnType { get; set; } + public PropertyModel() { } // For de-serializer from JSON + + [NonSerialized] private PythonPropertyType _property; + + public PropertyModel(IPythonPropertyType prop) : base(prop) { + ReturnType = prop.ReturnType.GetPersistentQualifiedName(); + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _property ?? (_property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), Documentation, declaringType, (Attributes & FunctionAttributes.Abstract) != 0)); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + _property.SetDocumentation(Documentation); + + var o = new PythonFunctionOverload(_property, mf.DefaultLocation); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(ReturnType), true); + _property.AddOverload(o); + } + } +} diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs new file mode 100644 index 000000000..e630d801f --- /dev/null +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -0,0 +1,55 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("TypeVar:{" + nameof(Name) + "}")] + internal sealed class TypeVarModel : MemberModel { + public string[] Constraints { get; set; } + public string Bound { get; set; } + public object Covariant { get; set; } + public object Contravariant { get; set; } + + public static TypeVarModel FromGeneric(IVariable v) { + var g = (IGenericTypeParameter)v.Value; + return new TypeVarModel { + Id = g.Name.GetStableHash(), + Name = g.Name, + QualifiedName = g.QualifiedName, + Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName()).ToArray(), + Bound = g.Bound?.QualifiedName, + Covariant = g.Covariant, + Contravariant = g.Contravariant + }; + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => new GenericTypeParameter(Name, mf.Module, + Constraints.Select(mf.ConstructType).ToArray(), + mf.ConstructType(Bound), Covariant, Contravariant, default); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + } +} diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs new file mode 100644 index 000000000..892dac8ac --- /dev/null +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -0,0 +1,59 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + [DebuggerDisplay("v:{Name} = {Value}")] + internal sealed class VariableModel : MemberModel { + public string Value { get; set; } + + public static VariableModel FromVariable(IVariable v) => new VariableModel { + Id = v.Name.GetStableHash(), + Name = v.Name, + QualifiedName = v.Name, + IndexSpan = v.Location.IndexSpan.ToModel(), + Value = v.Value.GetPersistentQualifiedName() + }; + + public static VariableModel FromInstance(string name, IPythonInstance inst) => new VariableModel { + Id = name.GetStableHash(), + Name = name, + QualifiedName = name, + Value = inst.GetPersistentQualifiedName() + }; + + public static VariableModel FromType(string name, IPythonType t) => new VariableModel { + Id = name.GetStableHash(), + Name = name, + QualifiedName = name, + IndexSpan = t.Location.IndexSpan.ToModel(), + Value = t.GetPersistentQualifiedName() + }; + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType; + return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default)); + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + } +} diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs new file mode 100644 index 000000000..53e6ab04b --- /dev/null +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -0,0 +1,256 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using LiteDB; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Collections; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Logging; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching { + internal sealed class ModuleDatabase : IModuleDatabaseService { + private const int DatabaseFormatVersion = 1; + + private readonly Dictionary _dependencies = new Dictionary(); + private readonly object _lock = new object(); + + private readonly IServiceContainer _services; + private readonly ILogger _log; + private readonly IFileSystem _fs; + private readonly string _databaseFolder; + + public ModuleDatabase(IServiceContainer services) { + _services = services; + _log = services.GetService(); + _fs = services.GetService(); + + var cfs = services.GetService(); + _databaseFolder = Path.Combine(cfs.CacheFolder, $"analysis.v{DatabaseFormatVersion}"); + } + + /// + /// Retrieves dependencies from the module persistent state. + /// + /// Python module to restore analysis for. + /// Python module dependency provider. + public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) { + dp = null; + + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; + } + + lock (_lock) { + if (_dependencies.TryGetValue(module.Name, out dp)) { + return true; + } + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + dp = new DependencyProvider(module, model); + _dependencies[module.Name] = dp; + return true; + } + } + return false; + } + + /// + /// Creates global scope from module persistent state. + /// Global scope is then can be used to construct module analysis. + /// + /// Python module to restore analysis for. + /// Python module global scope. + public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) { + gs = null; + + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; + } + + lock (_lock) { + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + gs = new RestoredGlobalScope(model, module); + } + } + + return gs != null; + } + + /// + /// Writes module data to the database. + /// + public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) + => Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken), cancellationToken); + + /// + /// Determines if module analysis exists in the storage. + /// + public bool ModuleExistsInStorage(string moduleName, string filePath) { + if (GetCachingLevel() == AnalysisCachingLevel.None) { + return false; + } + + for (var retries = 50; retries > 0; --retries) { + try { + lock (_lock) { + var dbPath = FindDatabaseFile(moduleName, filePath); + return !string.IsNullOrEmpty(dbPath); + } + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { + Thread.Sleep(10); + } + } + return false; + } + + public void Clear() { + lock (_lock) { + _dependencies.Clear(); + } + } + + private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { + var cachingLevel = GetCachingLevel(); + if (cachingLevel == AnalysisCachingLevel.None) { + return; + } + + var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); + if (model == null) { + // Caching level setting does not permit this module to be persisted. + return; + } + + Exception ex = null; + for (var retries = 50; retries > 0; --retries) { + lock (_lock) { + cancellationToken.ThrowIfCancellationRequested(); + try { + if (!_fs.DirectoryExists(_databaseFolder)) { + _fs.CreateDirectory(_databaseFolder); + } + + cancellationToken.ThrowIfCancellationRequested(); + using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) { + var modules = db.GetCollection("modules"); + modules.Upsert(model); + return; + } + } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { + ex = ex1; + Thread.Sleep(10); + } catch (Exception ex2) { + ex = ex2; + break; + } + } + } + + if (ex != null) { + _log?.Log(System.Diagnostics.TraceEventType.Warning, $"Unable to write analysis of {model.Name} to database. Exception {ex.Message}"); + if (ex.IsCriticalException()) { + throw ex; + } + } + } + + /// + /// Locates database file based on module information. Module is identified + /// by name, version, current Python interpreter version and/or hash of the + /// module content (typically file sizes). + /// + private string FindDatabaseFile(string moduleName, string filePath) { + var interpreter = _services.GetService(); + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel()); + if (string.IsNullOrEmpty(uniqueId)) { + return null; + } + + // Try module name as is. + var dbPath = Path.Combine(_databaseFolder, $"{uniqueId}.db"); + if (_fs.FileExists(dbPath)) { + return dbPath; + } + + // TODO: resolving to a different version can be an option + // Try with the major.minor Python version. + var pythonVersion = interpreter.Configuration.Version; + + dbPath = Path.Combine(_databaseFolder, $"{uniqueId}({pythonVersion.Major}.{pythonVersion.Minor}).db"); + if (_fs.FileExists(dbPath)) { + return dbPath; + } + + // Try with just the major Python version. + dbPath = Path.Combine(_databaseFolder, $"{uniqueId}({pythonVersion.Major}).db"); + return _fs.FileExists(dbPath) ? dbPath : null; + } + + private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) { + model = null; + + // We don't cache results here. Module resolution service decides when to call in here + // and it is responsible of overall management of the loaded Python modules. + for (var retries = 50; retries > 0; --retries) { + try { + // TODO: make combined db rather than per module? + var dbPath = FindDatabaseFile(moduleName, filePath); + if (string.IsNullOrEmpty(dbPath)) { + return false; + } + + using (var db = new LiteDatabase(dbPath)) { + if (!db.CollectionExists("modules")) { + return false; + } + + var modules = db.GetCollection("modules"); + model = modules.Find(m => m.Name == moduleName).FirstOrDefault(); + return model != null; + } + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { + Thread.Sleep(10); + } + } + return false; + } + private AnalysisCachingLevel GetCachingLevel() + => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; + + private sealed class DependencyProvider : IDependencyProvider { + private readonly ISet _dependencies; + + public DependencyProvider(IPythonModule module, ModuleModel model) { + var dc = new DependencyCollector(module); + dc.AddImports(model.Imports); + dc.AddFromImports(model.FromImports); + _dependencies = dc.Dependencies; + } + + public ISet GetDependencies(PythonAst ast) => _dependencies; + } + } +} diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs new file mode 100644 index 000000000..c66687872 --- /dev/null +++ b/src/Caching/Impl/ModuleFactory.cs @@ -0,0 +1,240 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Caching { + /// + /// Constructs module from its persistent model. + /// + internal sealed class ModuleFactory { + /// For use in tests so missing members will assert. + internal static bool EnableMissingMemberAssertions { get; set; } + + // TODO: better resolve circular references. + private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); + private readonly ModuleModel _model; + private readonly IGlobalScope _gs; + + public IPythonModule Module { get; } + public Location DefaultLocation { get; } + + public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) { + _model = model; + _gs = gs; + Module = module; + DefaultLocation = new Location(Module); + } + + public IPythonType ConstructType(string qualifiedName) + => ConstructMember(qualifiedName)?.GetPythonType(); + + public IMember ConstructMember(string qualifiedName) { + // Determine module name, member chain and if this is an instance. + if (!TypeNames.DeconstructQualifiedName(qualifiedName, out var parts)) { + return null; + } + + // See if member is a module first. + var module = GetModule(parts); + if (module == null) { + return null; + } + + var member = parts.ModuleName == Module.Name + ? GetMemberFromThisModule(parts.MemberNames) + : GetMemberFromModule(module, parts.MemberNames); + + if (parts.ObjectType != ObjectType.Instance) { + return member; + } + + var t = member.GetPythonType() ?? module.Interpreter.UnknownType; + return new PythonInstance(t); + } + + private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { + if (memberNames.Count == 0) { + return null; + } + + // Try from cache first + MemberModel currentModel = _model; + IMember m = null; + IPythonType declaringType = null; + + foreach (var name in memberNames) { + // Check if name has type arguments such as Union[int, str] + // Note that types can be nested like Union[int, Union[A, B]] + var memberName = name; + var typeArgs = GetTypeArguments(memberName, out var typeName); + if (!string.IsNullOrEmpty(typeName) && typeName != name) { + memberName = typeName; + } + + if (memberName == "") { + return null; + } + + var nextModel = currentModel.GetModel(memberName); + Debug.Assert(nextModel != null, $"Unable to find member {memberName} in module {Module.Name}"); + if (nextModel == null) { + return null; + } + + m = nextModel.Create(this, declaringType, _gs); + Debug.Assert(m != null); + + if (m is IGenericType gt && typeArgs.Count > 0) { + m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)); + } + + currentModel = nextModel; + declaringType = m as IPythonType; + Debug.Assert(declaringType != null); + if (declaringType == null) { + return null; + } + } + + return m; + } + + private IMember GetMemberFromModule(IPythonModule module, IReadOnlyList memberNames) + => memberNames.Count == 0 ? module : GetMember(module, memberNames); + + + private IPythonModule GetModule(QualifiedNameParts parts) { + if (parts.ModuleName == Module.Name) { + return Module; + } + using (_moduleReentrancy.Push(parts.ModuleName, out var reentered)) { + if (reentered) { + return null; + } + // Here we do not call GetOrLoad since modules references here must + // either be loaded already since they were required to create + // persistent state from analysis. Also, occasionally types come + // from the stub and the main module was never loaded. This, for example, + // happens with io which has member with mmap type coming from mmap + // stub rather than the primary mmap module. + var m = parts.IsStub + ? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName) + : Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); + + if (m != null) { + return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(m) : m; + } + return null; + } + } + + private IMember GetMember(IMember root, IEnumerable memberNames) { + var member = root; + foreach (var n in memberNames) { + var memberName = n; + // Check if name has type arguments such as Union[int, str] + // Note that types can be nested like Union[int, Union[A, B]] + var typeArgs = GetTypeArguments(memberName, out var typeName); + if (!string.IsNullOrEmpty(typeName) && typeName != memberName) { + memberName = typeName; + } + + var mc = member as IMemberContainer; + Debug.Assert(mc != null); + + if (mc is IBuiltinsPythonModule builtins) { + // Builtins require special handling since there may be 'hidden' names + // like __NoneType__ which need to be mapped to visible types. + member = GetBuiltinMember(builtins, memberName) ?? builtins.Interpreter.UnknownType; + } else { + member = mc?.GetMember(memberName); + // Work around problem that some stubs have incorrectly named tuples. + // For example, in posix.pyi variable for the named tuple is not named as the tuple: + // sched_param = NamedTuple('sched_priority', [('sched_priority', int),]) + member = member ?? (mc as PythonModule)?.GlobalScope.Variables + .FirstOrDefault(v => v.Value is ITypingNamedTupleType nt && nt.Name == memberName); + } + + if (member == null) { + var containerName = mc is IPythonType t ? t.Name : ""; + Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); + break; + } + + member = typeArgs.Count > 0 && member is IGenericType gt + ? gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)) + : member; + } + + return member; + } + + private IMember GetBuiltinMember(IBuiltinsPythonModule builtins, string memberName) { + if (memberName.StartsWithOrdinal("__")) { + memberName = memberName.Substring(2, memberName.Length - 4); + } + + switch (memberName) { + case "NoneType": + return builtins.Interpreter.GetBuiltinType(BuiltinTypeId.NoneType); + case "Unknown": + return builtins.Interpreter.UnknownType; + } + return builtins.GetMember(memberName); + } + + private IReadOnlyList GetTypeArguments(string memberName, out string typeName) { + typeName = null; + // TODO: better handle generics. + // https://github.com/microsoft/python-language-server/issues/1215 + // Determine generic type arguments, if any, so we can construct + // complex types from parts, such as Union[typing.Any, a.b.c]. + var typeArgs = new List(); + var openBracket = memberName.IndexOf('['); + if (openBracket > 0) { + var closeBracket = memberName.LastIndexOf(']'); + if (closeBracket > 0) { + var argumentString = memberName.Substring(openBracket + 1, closeBracket - openBracket - 1); + // Extract type names from argument string. Note that types themselves + // can have arguments: Union[int, Union[int, Union[str, bool]], ...]. + var qualifiedNames = TypeNames.GetTypeNames(argumentString, ','); + foreach (var qn in qualifiedNames) { + var t = ConstructType(qn); + if (t == null) { + TypeNames.DeconstructQualifiedName(qn, out var parts); + typeName = string.Join(".", parts.MemberNames); + t = new GenericTypeParameter(typeName, Module, Array.Empty(), null, null, null, default); + } + typeArgs.Add(t); + } + typeName = memberName.Substring(0, openBracket); + } + } + return typeArgs; + } + } +} diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs new file mode 100644 index 000000000..51762ecfe --- /dev/null +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -0,0 +1,119 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; + +namespace Microsoft.Python.Analysis.Caching { + internal static class ModuleUniqueId { + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel) + => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); + + public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel) { + if(cachingLevel == AnalysisCachingLevel.None) { + return null; + } + if (moduleType == ModuleType.User) { + // Only for tests. + return $"{moduleName}"; + } + + var interpreter = services.GetService(); + var fs = services.GetService(); + + var moduleResolution = interpreter.ModuleResolution; + var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs); + switch(modulePathType) { + case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library: + return null; + case PythonLibraryPathType.StdLib when cachingLevel < AnalysisCachingLevel.System: + return null; + } + + if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site) { + // Module can be a submodule of a versioned package. In this case we want to use + // version of the enclosing package so we have to look up the chain of folders. + var moduleRootName = moduleName.Split('.')[0]; + var moduleFilesFolder = Path.GetDirectoryName(filePath); + var installationFolder = Path.GetDirectoryName(moduleFilesFolder); + + var versionFolder = installationFolder; + while (!string.IsNullOrEmpty(versionFolder)) { + // If module is in site-packages and is versioned, then unique id = name + version + interpreter version. + // Example: 'requests' and 'requests-2.21.0.dist-info'. + // TODO: for egg (https://github.com/microsoft/python-language-server/issues/196), consider *.egg-info + var folders = fs.GetFileSystemEntries(versionFolder, "*-*.dist-info", SearchOption.TopDirectoryOnly) + .Select(Path.GetFileName) + .Where(n => n.StartsWith(moduleRootName, StringComparison.OrdinalIgnoreCase)) // Module name can be capitalized differently. + .ToArray(); + + if (folders.Length == 1) { + var fileName = Path.GetFileNameWithoutExtension(folders[0]); + var dash = fileName.IndexOf('-'); + return $"{moduleName}({fileName.Substring(dash + 1)})"; + } + // Move up if nothing is found. + versionFolder = Path.GetDirectoryName(versionFolder); + } + } + + var config = interpreter.Configuration; + if (moduleType.IsCompiled() || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) { + // If module is a standard library, unique id is its name + interpreter version. + return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; + } + + var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); + var hash = HashModuleFileSizes(parent); + // If all else fails, hash modules file sizes. + return $"{moduleName}.{(ulong)hash}"; + } + + private static long HashModuleFileSizes(IImportChildrenSource source) { + var hash = 0L; + var names = source.GetChildrenNames(); + foreach (var name in names) { + if (source.TryGetChildImport(name, out var child)) { + if (child is ModuleImport moduleImport) { + hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); + } + + if (child is IImportChildrenSource childSource) { + hash = unchecked(hash * 31 ^ HashModuleFileSizes(childSource)); + } + } + } + + return hash; + } + + private static PythonLibraryPathType GetModulePathType(string modulePath, IEnumerable libraryPaths, IFileSystem fs) { + if (string.IsNullOrEmpty(modulePath)) { + return PythonLibraryPathType.Unspecified; + } + return libraryPaths + .OrderByDescending(p => p.Path.Length) + .FirstOrDefault(p => fs.IsPathUnderRoot(p.Path, modulePath))?.Type ?? PythonLibraryPathType.Unspecified; + } + } +} diff --git a/src/Caching/Impl/Properties/AssemblyInfo.cs b/src/Caching/Impl/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d2256d3fe --- /dev/null +++ b/src/Caching/Impl/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Caching.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs new file mode 100644 index 000000000..8da9634ed --- /dev/null +++ b/src/Caching/Impl/PythonDbModule.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.Analysis.Caching { + internal sealed class PythonDbModule : SpecializedModule { + private readonly NewLineLocation[] _newLines; + private readonly int _fileSize; + + public PythonDbModule(ModuleModel model, string filePath, IServiceContainer services) + : base(model.Name, filePath, services) { + Documentation = model.Documentation; + _newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray(); + _fileSize = model.FileSize; + } + + public void Construct(ModuleModel model) { + var gs = new RestoredGlobalScope(model, this); + GlobalScope = gs; + gs.ReconstructVariables(); + } + + protected override string LoadContent() => string.Empty; + + public override string Documentation { get; } + public override IEnumerable GetMemberNames() => GlobalScope.Variables.Names; + + #region ILocationConverter + public override SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(_newLines, index); + public override int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(_newLines, location, _fileSize); + #endregion + } +} diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs new file mode 100644 index 000000000..4d1d33539 --- /dev/null +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -0,0 +1,37 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; + +namespace Microsoft.Python.Analysis.Caching { + public enum ObjectType { + Type, + Instance, + Module, + VariableModule, + BuiltinModule + } + + internal struct QualifiedNameParts { + /// Object type. + public ObjectType ObjectType; + /// Module name. + public string ModuleName; + /// Indicates if module is a stub. + public bool IsStub; + /// Module member names such as 'A', 'B', 'C' from module:A.B.C. + public IReadOnlyList MemberNames; + } +} diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs new file mode 100644 index 000000000..77e3bb255 --- /dev/null +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -0,0 +1,88 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching { + internal sealed class RestoredGlobalScope : IRestoredGlobalScope { + private readonly VariableCollection _scopeVariables = new VariableCollection(); + private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. + private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. + + public RestoredGlobalScope(ModuleModel model, IPythonModule module) { + _model = model ?? throw new ArgumentNullException(nameof(model)); + Module = module ?? throw new ArgumentNullException(nameof(module)); + Name = model.Name; + _factory = new ModuleFactory(_model, Module, this); + DeclareVariables(); + } + + public void ReconstructVariables() { + var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + foreach (var m in models.Concat(_model.Variables)) { + m.Populate(_factory, null, this); + } + // TODO: re-declare __doc__, __name__, etc. +#if !DEBUG + _model = null; + _factory = null; +#endif + } + + private void DeclareVariables() { + // Member creation may be non-linear. Consider function A returning instance + // of a class or type info of a function which hasn't been created yet. + // Thus first create members so we can find then, then populate them with content. + var mf = new ModuleFactory(_model, Module, this); + + // Generics first + var typeVars = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + foreach (var m in typeVars) { + _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation); + } + + // Declare variables in the order of appearance since later variables + // may use types declared in the preceding ones. + foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { + var v = (IVariable)vm.Create(mf, null, this); + _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); + } + } + + #region IScope + public string Name { get; } + public ScopeStatement Node => null; + public IScope OuterScope => null; + public IReadOnlyList Children => Array.Empty(); + public IEnumerable EnumerateTowardsGlobal => Enumerable.Empty(); + public IEnumerable EnumerateFromGlobal => Enumerable.Empty(); + public IVariableCollection Variables => _scopeVariables; + public IVariableCollection NonLocals => VariableCollection.Empty; + public IVariableCollection Globals => VariableCollection.Empty; + public IVariableCollection Imported => VariableCollection.Empty; + public IPythonModule Module { get; } + IGlobalScope IScope.GlobalScope => this; + + public void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) { } + public void LinkVariable(string name, IVariable v, Location location) => throw new NotImplementedException() { }; + #endregion + } +} diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs new file mode 100644 index 000000000..2d12f482a --- /dev/null +++ b/src/Caching/Impl/TypeNames.cs @@ -0,0 +1,163 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Caching { + internal static class TypeNames { + /// + /// Constructs persistent member name based on the member and the current module. + /// Persistent name contains complete information for the member restoration code. + /// + public static string GetPersistentQualifiedName(this IMember m) { + var t = m.GetPythonType(); + if (!t.IsUnknown()) { + switch (m) { + case IPythonInstance _: // constants and strings map here. + return $"i:{t.QualifiedName}"; + case IBuiltinsPythonModule b: + return $"b:{b.QualifiedName}"; + case PythonVariableModule vm: + return $"p:{vm.QualifiedName}"; + case IPythonModule mod: + return $"m:{mod.QualifiedName}"; + case IPythonType pt when pt.DeclaringModule.ModuleType == ModuleType.Builtins: + return $"t:{(pt.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : pt.QualifiedName)}"; + case IPythonType pt: + return $"t:{pt.QualifiedName}"; + case null: + break; + } + } + return null; + } + + /// + /// Splits qualified type name in form of i:A(3.6).B.C into parts. as well as determines if + /// qualified name designates instance (prefixed with 'i:'). + /// + /// Qualified name to split. May include instance prefix. + /// Qualified name parts. + public static bool DeconstructQualifiedName(string qualifiedName, out QualifiedNameParts parts) { + parts = new QualifiedNameParts(); + if (string.IsNullOrEmpty(qualifiedName)) { + return false; + } + + GetObjectTypeFromPrefix(qualifiedName, ref parts, out var prefixOffset); + GetModuleNameAndMembers(qualifiedName, ref parts, prefixOffset); + + return !string.IsNullOrEmpty(parts.ModuleName); + } + + private static void GetObjectTypeFromPrefix(string qualifiedName, ref QualifiedNameParts parts, out int prefixOffset) { + prefixOffset = 2; + if (qualifiedName.StartsWith("i:")) { + parts.ObjectType = ObjectType.Instance; + } else if (qualifiedName.StartsWith("m:")) { + parts.ObjectType = ObjectType.Module; + } else if (qualifiedName.StartsWith("p:")) { + parts.ObjectType = ObjectType.VariableModule; + } else if (qualifiedName.StartsWith("b:")) { + parts.ObjectType = ObjectType.BuiltinModule; + } else if (qualifiedName.StartsWith("t:")) { + parts.ObjectType = ObjectType.Type; + } else { + // Unprefixed name is typically an argument to another type like Union[int, typing:Any] + parts.ObjectType = ObjectType.Type; + prefixOffset = 0; + } + } + + private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedNameParts parts, int prefixOffset) { + // Strip the prefix, turning i:module:A.B.C into module:A.B.C + var typeName = qualifiedName.Substring(prefixOffset); + + var moduleSeparatorIndex = typeName.IndexOf(':'); + if (moduleSeparatorIndex < 0) { + switch (parts.ObjectType) { + case ObjectType.Type: + case ObjectType.Instance: + // No module name means built-in type like 'int' or 'i:str'. + parts.ModuleName = @"builtins"; + parts.MemberNames = typeName == "..." ? new[] { "ellipsis" } : typeName.Split('.').ToArray(); + break; + default: + parts.ModuleName = typeName; + parts.MemberNames = Array.Empty(); + DetermineModuleType(ref parts); + break; + } + return; + } + + // Extract module name and member names, of any. + parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex); + var memberNamesOffset = parts.ModuleName.Length + 1; + parts.MemberNames = GetTypeNames(typeName.Substring(memberNamesOffset), '.'); + + DetermineModuleType(ref parts); + } + + private static void DetermineModuleType(ref QualifiedNameParts parts) { + if (parts.ModuleName.EndsWith("(stub)")) { + parts.ModuleName = parts.ModuleName.Substring(0, parts.ModuleName.Length - 6); + parts.IsStub = true; + } + } + + public static IReadOnlyList GetTypeNames(string qualifiedTypeName, char separator) { + var parts = new List(); + for (var i = 0; i < qualifiedTypeName.Length; i++) { + var part = GetTypeName(qualifiedTypeName, ref i, separator); + if (string.IsNullOrEmpty(part)) { + break; + } + parts.Add(part.Trim()); + } + return parts; + } + + public static string GetTypeName(string s, ref int i, char separator) { + var braceCounter = new Stack(); + var start = i; + for (; i < s.Length; i++) { + var ch = s[i]; + + if (ch == '[') { + braceCounter.Push(ch); + continue; + } + + if (ch == ']') { + if (braceCounter.Count > 0) { + braceCounter.Pop(); + } + } + + if (braceCounter.Count == 0 && ch == separator) { + break; + } + } + + return s.Substring(start, i - start).Trim(); + } + } +} diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs new file mode 100644 index 000000000..8baa49828 --- /dev/null +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -0,0 +1,94 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Tests; +using Microsoft.Python.Analysis.Types; +using Newtonsoft.Json; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + public abstract class AnalysisCachingTestBase: AnalysisTestBase { + protected AnalysisCachingTestBase() { + ModuleFactory.EnableMissingMemberAssertions = true; + } + + protected static string ToJson(object model) { + var sb = new StringBuilder(); + using (var sw = new StringWriter(sb)) + using (var jw = new JsonTextWriter(sw)) { + jw.Formatting = Formatting.Indented; + var js = new JsonSerializer(); + js.Serialize(jw, model); + } + return sb.ToString(); + } + + protected string BaselineFilesFolder { + get { + var testAssembly = Assembly.GetExecutingAssembly().GetAssemblyPath(); + var outDirectory = Path.GetDirectoryName(testAssembly); + return Path.GetFullPath(Path.Combine(outDirectory, "..", "..", "..", "src", "Caching", "Test", "Files")); + } + } + + protected string GetBaselineFileName(string testName, string suffix = null) + => Path.ChangeExtension(suffix == null + ? Path.Combine(BaselineFilesFolder, testName) + : Path.Combine(BaselineFilesFolder, testName + suffix), "json"); + + internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { + var dbModule = new PythonDbModule(model, modulePath, Services); + dbModule.Construct(model); + return dbModule; + } + + internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + // In real case dependency analysis will restore model dependencies. + // Here we don't go through the dependency analysis so we have to + // manually restore dependent modules. + var dc = new DependencyCollector(m); + dc.AddImports(model.Imports); + dc.AddFromImports(model.FromImports); + foreach(var dep in dc.Dependencies) { + m.Interpreter.ModuleResolution.GetOrLoadModule(dep.Name); + } + + var dcs = new DependencyCollector(m, true); + dcs.AddImports(model.StubImports); + dcs.AddFromImports(model.StubFromImports); + foreach (var dep in dcs.Dependencies) { + m.Interpreter.TypeshedResolution.GetOrLoadModule(dep.Name); + } + + var analyzer = Services.GetService(); + await analyzer.WaitForCompleteAnalysisAsync(); + + using (var dbModule = CreateDbModule(model, m.FilePath)) { + dbModule.Should().HaveSameMembersAs(m); + } + } + } +} diff --git a/src/Caching/Test/AssemblySetup.cs b/src/Caching/Test/AssemblySetup.cs new file mode 100644 index 000000000..4e92972b6 --- /dev/null +++ b/src/Caching/Test/AssemblySetup.cs @@ -0,0 +1,34 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core.Testing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public sealed class AssemblySetup { + [AssemblyInitialize] + public static void Initialize(TestContext testContext) => AnalysisTestEnvironment.Initialize(); + + private class AnalysisTestEnvironment : TestEnvironmentImpl, ITestEnvironment { + public static void Initialize() { + var instance = new AnalysisTestEnvironment(); + Instance = instance; + TestEnvironment.Current = instance; + } + } + } +} diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs new file mode 100644 index 000000000..b046213fb --- /dev/null +++ b/src/Caching/Test/ClassesTests.cs @@ -0,0 +1,154 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public class ClassesTests : AnalysisCachingTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + private string BaselineFileName => GetBaselineFileName(TestContext.TestName); + + [TestMethod, Priority(0)] + public async Task NestedClasses() { + const string code = @" +x = 'str' + +class A: + def methodA(self): + return True + +class B: + x: int + + class C: + def __init__(self): + self.y = 1 + def methodC(self): + return False + + def methodB1(self): + return self.C() + + def methodB2(self): + return self.C().y + +c = B().methodB1() +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + Baseline.CompareToFile(BaselineFileName, json); + } + + [TestMethod, Priority(0)] + public async Task ForwardDeclarations() { + const string code = @" +x = 'str' + +class A: + def methodA1(self): + return B() + + def methodA2(self): + return func() + +class B: + class C: + def methodC(self): + return func() + + def methodB1(self): + def a(): + return 1 + return a + +def func(): + return 1 + +a = B().methodB1() +b = A().methodA1() +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("a").Which.Should().HaveType("a"); + analysis.Should().HaveVariable("b").Which.Should().HaveType("B"); + + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { + dbModule.Should().HaveSameMembersAs(analysis.Document); + } + } + + [TestMethod, Priority(0)] + public async Task GenericClass() { + const string code = @" +from typing import Generic, TypeVar, Dict + +K = TypeVar('K') +V = TypeVar('V') + +class A(Generic[K, V], Dict[K, V]): + def key(self) -> K: + return K + + def value(self): + return V + +x = A(1, 'a') +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { + dbModule.Should().HaveSameMembersAs(analysis.Document); + } + } + + [TestMethod, Priority(0)] + public async Task ClassOwnDocumentation() { + const string code = @" +class A: + '''class A doc''' + +class B(A): + def __init__(self): + '''__init__ doc''' + return +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + // In JSON, class A should have 'class A doc' documentation while B should have none. + Baseline.CompareToFile(BaselineFileName, json); + } + } +} diff --git a/src/Caching/Test/CoreTests.cs b/src/Caching/Test/CoreTests.cs new file mode 100644 index 000000000..4b1adaafa --- /dev/null +++ b/src/Caching/Test/CoreTests.cs @@ -0,0 +1,114 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public class CoreTests : AnalysisCachingTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + private string BaselineFileName => GetBaselineFileName(TestContext.TestName); + private string GetBaselineFileNameWithSuffix(string suffix) => GetBaselineFileName(TestContext.TestName, suffix); + + [TestMethod, Priority(0)] + public async Task SmokeTest() { + const string code = @" +x = 'str' + +class C: + x: int + def __init__(self): + self.y = 1 + + def method(self): + return func() + + @property + def prop(self) -> int: + return x + +def func(): + return 2.0 + +c = C() +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + Baseline.CompareToFile(BaselineFileName, json); + } + + [DataTestMethod, Priority(0)] + [DataRow("t:str", "builtins", "str", ObjectType.Type)] + [DataRow("i:str", "builtins", "str", ObjectType.Instance)] + [DataRow("i:...", "builtins", "ellipsis", ObjectType.Instance)] + [DataRow("t:ellipsis", "builtins", "ellipsis", ObjectType.Type)] + [DataRow("i:builtins:str", "builtins", "str", ObjectType.Instance)] + [DataRow("i:mod:x", "mod", "x", ObjectType.Instance)] + [DataRow("t:typing:Union[str, tuple]", "typing", "Union[str, tuple]", ObjectType.Type)] + [DataRow("t:typing:Union[typing:Any, mod:y]", "typing", "Union[typing:Any, mod:y]", ObjectType.Type)] + [DataRow("t:typing:Union[typing:Union[str, int], mod:y]", "typing", "Union[typing:Union[str, int], mod:y]", ObjectType.Type)] + [DataRow("m:typing", "typing", "", ObjectType.Module)] + [DataRow("p:A", "A", "", ObjectType.VariableModule)] + public void QualifiedNames(string qualifiedName, string moduleName, string typeName, ObjectType objectType) { + TypeNames.DeconstructQualifiedName(qualifiedName, out var parts); + parts.ModuleName.Should().Be(moduleName); + switch (objectType) { + case ObjectType.Instance: + case ObjectType.Type: + parts.MemberNames[0].Should().Be(typeName); + break; + default: + parts.MemberNames.Should().BeEmpty(); + break; + } + parts.ObjectType.Should().Be(objectType); + } + + [DataTestMethod, Priority(0)] + [DataRow(true)] + [DataRow(false)] + public async Task VersionHandling(bool is3x) { + const string code = @" +if sys.version_info >= (3, 0): + def func(a, b, c): ... +else: + def func(a): ... +"; + var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); + analysis.Should().HaveFunction("func") + .Which.Should().HaveSingleOverload() + .Which.Should().HaveParameters(is3x ? new[] { "a", "b", "c" } : new[] { "a" }); + + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + Baseline.CompareToFile(GetBaselineFileNameWithSuffix(is3x ? "3" : "2"), json); + } + } +} diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json new file mode 100644 index 000000000..647537712 --- /dev/null +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -0,0 +1,202 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [ + { + "Documentation": "class A doc", + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 778, + "Name": "A", + "QualifiedName": "module:A", + "IndexSpan": { + "Start": 8, + "Length": 1 + } + }, + { + "Documentation": null, + "Bases": [ + "t:module:A", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "QualifiedName": "module:B.__init__", + "IndexSpan": { + "Start": 58, + "Length": 8 + } + } + ], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 779, + "Name": "B", + "QualifiedName": "module:B", + "IndexSpan": { + "Start": 43, + "Length": 1 + } + } + ], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 12, + "Kind": 3 + }, + { + "EndIndex": 35, + "Kind": 3 + }, + { + "EndIndex": 37, + "Kind": 3 + }, + { + "EndIndex": 50, + "Kind": 3 + }, + { + "EndIndex": 75, + "Kind": 3 + }, + { + "EndIndex": 99, + "Kind": 3 + }, + { + "EndIndex": 115, + "Kind": 3 + } + ], + "FileSize": 115, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/MemberLocations.json b/src/Caching/Test/Files/MemberLocations.json new file mode 100644 index 000000000..91c00690d --- /dev/null +++ b/src/Caching/Test/Files/MemberLocations.json @@ -0,0 +1,373 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "a", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "b", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 799444, + "Name": "sum", + "QualifiedName": "module:sum", + "IndexSpan": { + "Start": 19, + "Length": 3 + } + } + ], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": { + "Start": 2, + "Length": 1 + } + } + ], + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 935009768, + "Name": "methodB2", + "QualifiedName": "module:B.methodB2", + "IndexSpan": { + "Start": 253, + "Length": 8 + } + } + ], + "Properties": [ + { + "ReturnType": "i:int", + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -947452202, + "Name": "propertyB", + "QualifiedName": "module:B.propertyB", + "IndexSpan": { + "Start": 207, + "Length": 9 + } + } + ], + "Fields": [ + { + "Value": "i:int", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": null + } + ], + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B.C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "QualifiedName": "module:B.C.__init__", + "IndexSpan": { + "Start": 101, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B.C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1909501045, + "Name": "methodC", + "QualifiedName": "module:B.C.methodC", + "IndexSpan": { + "Start": 148, + "Length": 7 + } + } + ], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 780, + "Name": "C", + "QualifiedName": "module:B.C", + "IndexSpan": { + "Start": 85, + "Length": 1 + } + } + ], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 779, + "Name": "B", + "QualifiedName": "module:B", + "IndexSpan": { + "Start": 57, + "Length": 1 + } + } + ], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 13, + "Kind": 3 + }, + { + "EndIndex": 15, + "Kind": 3 + }, + { + "EndIndex": 31, + "Kind": 3 + }, + { + "EndIndex": 49, + "Kind": 3 + }, + { + "EndIndex": 51, + "Kind": 3 + }, + { + "EndIndex": 61, + "Kind": 3 + }, + { + "EndIndex": 73, + "Kind": 3 + }, + { + "EndIndex": 75, + "Kind": 3 + }, + { + "EndIndex": 89, + "Kind": 3 + }, + { + "EndIndex": 118, + "Kind": 3 + }, + { + "EndIndex": 136, + "Kind": 3 + }, + { + "EndIndex": 164, + "Kind": 3 + }, + { + "EndIndex": 182, + "Kind": 3 + }, + { + "EndIndex": 184, + "Kind": 3 + }, + { + "EndIndex": 199, + "Kind": 3 + }, + { + "EndIndex": 225, + "Kind": 3 + }, + { + "EndIndex": 243, + "Kind": 3 + }, + { + "EndIndex": 245, + "Kind": 3 + }, + { + "EndIndex": 270, + "Kind": 3 + }, + { + "EndIndex": 288, + "Kind": 3 + } + ], + "FileSize": 288, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json new file mode 100644 index 000000000..efb7c122d --- /dev/null +++ b/src/Caching/Test/Files/NestedClasses.json @@ -0,0 +1,424 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": { + "Start": 2, + "Length": 1 + } + }, + { + "Value": "i:module:B.C", + "Id": 812, + "Name": "c", + "QualifiedName": "c", + "IndexSpan": { + "Start": 333, + "Length": 1 + } + } + ], + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:A", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:bool" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1909501047, + "Name": "methodA", + "QualifiedName": "module:A.methodA", + "IndexSpan": { + "Start": 33, + "Length": 7 + } + } + ], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 778, + "Name": "A", + "QualifiedName": "module:A", + "IndexSpan": { + "Start": 21, + "Length": 1 + } + }, + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:module:B.C" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 935009767, + "Name": "methodB1", + "QualifiedName": "module:B.methodB1", + "IndexSpan": { + "Start": 235, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 935009768, + "Name": "methodB2", + "QualifiedName": "module:B.methodB2", + "IndexSpan": { + "Start": 287, + "Length": 8 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": null + } + ], + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B.C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "QualifiedName": "module:B.C.__init__", + "IndexSpan": { + "Start": 122, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:B.C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:bool" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1909501045, + "Name": "methodC", + "QualifiedName": "module:B.C.methodC", + "IndexSpan": { + "Start": 175, + "Length": 7 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 834, + "Name": "y", + "QualifiedName": "y", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 780, + "Name": "C", + "QualifiedName": "module:B.C", + "IndexSpan": { + "Start": 106, + "Length": 1 + } + } + ], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 779, + "Name": "B", + "QualifiedName": "module:B", + "IndexSpan": { + "Start": 78, + "Length": 1 + } + } + ], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 13, + "Kind": 3 + }, + { + "EndIndex": 15, + "Kind": 3 + }, + { + "EndIndex": 25, + "Kind": 3 + }, + { + "EndIndex": 49, + "Kind": 3 + }, + { + "EndIndex": 70, + "Kind": 3 + }, + { + "EndIndex": 72, + "Kind": 3 + }, + { + "EndIndex": 82, + "Kind": 3 + }, + { + "EndIndex": 94, + "Kind": 3 + }, + { + "EndIndex": 96, + "Kind": 3 + }, + { + "EndIndex": 110, + "Kind": 3 + }, + { + "EndIndex": 139, + "Kind": 3 + }, + { + "EndIndex": 163, + "Kind": 3 + }, + { + "EndIndex": 191, + "Kind": 3 + }, + { + "EndIndex": 217, + "Kind": 3 + }, + { + "EndIndex": 227, + "Kind": 3 + }, + { + "EndIndex": 252, + "Kind": 3 + }, + { + "EndIndex": 277, + "Kind": 3 + }, + { + "EndIndex": 279, + "Kind": 3 + }, + { + "EndIndex": 304, + "Kind": 3 + }, + { + "EndIndex": 331, + "Kind": 3 + }, + { + "EndIndex": 333, + "Kind": 3 + }, + { + "EndIndex": 353, + "Kind": 3 + } + ], + "FileSize": 353, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json new file mode 100644 index 000000000..3c94c2dc5 --- /dev/null +++ b/src/Caching/Test/Files/SmokeTest.json @@ -0,0 +1,321 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [ + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": "i:float" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 24395611, + "Name": "func", + "QualifiedName": "module:func", + "IndexSpan": { + "Start": 207, + "Length": 4 + } + } + ], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": { + "Start": 2, + "Length": 1 + } + }, + { + "Value": "i:module:C", + "Id": 812, + "Name": "c", + "QualifiedName": "c", + "IndexSpan": { + "Start": 234, + "Length": 1 + } + } + ], + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "QualifiedName": "module:C.__init__", + "IndexSpan": { + "Start": 45, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:module:C", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:float" + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -2139806792, + "Name": "method", + "QualifiedName": "module:C.method", + "IndexSpan": { + "Start": 100, + "Length": 6 + } + } + ], + "Properties": [ + { + "ReturnType": "i:int", + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 24690682, + "Name": "prop", + "QualifiedName": "module:C.prop", + "IndexSpan": { + "Start": 163, + "Length": 4 + } + } + ], + "Fields": [ + { + "Value": "i:int", + "Id": 833, + "Name": "x", + "QualifiedName": "x", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": 834, + "Name": "y", + "QualifiedName": "y", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 780, + "Name": "C", + "QualifiedName": "module:C", + "IndexSpan": { + "Start": 21, + "Length": 1 + } + } + ], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 13, + "Kind": 3 + }, + { + "EndIndex": 15, + "Kind": 3 + }, + { + "EndIndex": 25, + "Kind": 3 + }, + { + "EndIndex": 37, + "Kind": 3 + }, + { + "EndIndex": 62, + "Kind": 3 + }, + { + "EndIndex": 82, + "Kind": 3 + }, + { + "EndIndex": 92, + "Kind": 3 + }, + { + "EndIndex": 115, + "Kind": 3 + }, + { + "EndIndex": 138, + "Kind": 3 + }, + { + "EndIndex": 140, + "Kind": 3 + }, + { + "EndIndex": 155, + "Kind": 3 + }, + { + "EndIndex": 183, + "Kind": 3 + }, + { + "EndIndex": 201, + "Kind": 3 + }, + { + "EndIndex": 203, + "Kind": 3 + }, + { + "EndIndex": 216, + "Kind": 3 + }, + { + "EndIndex": 232, + "Kind": 3 + }, + { + "EndIndex": 234, + "Kind": 3 + }, + { + "EndIndex": 243, + "Kind": 3 + } + ], + "FileSize": 243, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json new file mode 100644 index 000000000..ee4d02e74 --- /dev/null +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -0,0 +1,148 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "a", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 24395611, + "Name": "func", + "QualifiedName": "module:func", + "IndexSpan": { + "Start": 77, + "Length": 4 + } + } + ], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 34, + "Kind": 3 + }, + { + "EndIndex": 62, + "Kind": 3 + }, + { + "EndIndex": 69, + "Kind": 3 + }, + { + "EndIndex": 91, + "Kind": 3 + } + ], + "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json new file mode 100644 index 000000000..1d92c7ff2 --- /dev/null +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -0,0 +1,160 @@ +{ + "UniqueId": "module", + "Documentation": "", + "Functions": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "a", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "b", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "c", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 24395611, + "Name": "func", + "QualifiedName": "module:func", + "IndexSpan": { + "Start": 42, + "Length": 4 + } + } + ], + "Variables": [ + { + "Value": "t:bool", + "Id": -529376420, + "Name": "__debug__", + "QualifiedName": "__debug__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": -1636005055, + "Name": "__doc__", + "QualifiedName": "__doc__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 875442003, + "Name": "__file__", + "QualifiedName": "__file__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 1097116834, + "Name": "__name__", + "QualifiedName": "__name__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:str", + "Id": 75395663, + "Name": "__package__", + "QualifiedName": "__package__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:list", + "Id": 1154586556, + "Name": "__path__", + "QualifiedName": "__path__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:dict", + "Id": 817929997, + "Name": "__dict__", + "QualifiedName": "__dict__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:object", + "Id": 1253875154, + "Name": "__spec__", + "QualifiedName": "__spec__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [], + "TypeVars": [], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 2, + "Kind": 3 + }, + { + "EndIndex": 34, + "Kind": 3 + }, + { + "EndIndex": 62, + "Kind": 3 + }, + { + "EndIndex": 69, + "Kind": 3 + }, + { + "EndIndex": 91, + "Kind": 3 + } + ], + "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], + "Id": -2131035837, + "Name": "module", + "QualifiedName": "module", + "IndexSpan": null +} \ No newline at end of file diff --git a/src/Caching/Test/FluentAssertions/AssertionsFactory.cs b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs new file mode 100644 index 000000000..594897a40 --- /dev/null +++ b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs @@ -0,0 +1,37 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.Analysis.Caching.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal static class AssertionsFactory { + public static DocumentAnalysisAssertions Should(this IDocumentAnalysis analysis) => new DocumentAnalysisAssertions(analysis); + public static MemberAssertions Should(this IMember member) => new MemberAssertions(member); + public static PythonFunctionAssertions Should(this IPythonFunctionType f) => new PythonFunctionAssertions(f); + public static PythonFunctionOverloadAssertions Should(this IPythonFunctionOverload f) => new PythonFunctionOverloadAssertions(f); + public static ParameterAssertions Should(this IParameterInfo p) => new ParameterAssertions(p); + + public static RangeAssertions Should(this Range? range) => new RangeAssertions(range); + + public static SourceSpanAssertions Should(this SourceSpan span) => new SourceSpanAssertions(span); + public static SourceSpanAssertions Should(this SourceSpan? span) => new SourceSpanAssertions(span.Value); + public static VariableAssertions Should(this IVariable v) => new VariableAssertions(v); + } +} diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs new file mode 100644 index 000000000..0d0183579 --- /dev/null +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -0,0 +1,318 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public class LibraryModulesTests : AnalysisCachingTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + private string BaselineFileName => GetBaselineFileName(TestContext.TestName); + + [TestMethod, Priority(0)] + [Ignore("Builtins module have custom member handling. We do not persist it yet.")] + public async Task Builtins() { + var analysis = await GetAnalysisAsync(string.Empty); + var builtins = analysis.Document.Interpreter.ModuleResolution.BuiltinsModule; + var model = ModuleModel.FromAnalysis(builtins.Analysis, Services, AnalysisCachingLevel.Library); + + var json = ToJson(model); + Baseline.CompareToFile(BaselineFileName, json); + + var dbModule = new PythonDbModule(model, null, Services); + dbModule.Should().HaveSameMembersAs(builtins); + } + + [TestMethod, Priority(0)] + public Task Ast() => TestModule("ast"); + + [TestMethod, Priority(0)] + public Task Asyncio() => TestModule("asyncio"); + + [TestMethod, Priority(0)] + public Task Base64() => TestModule("base64"); + + [TestMethod, Priority(0)] + public Task Bisect() => TestModule("bisect"); + + [TestMethod, Priority(0)] + public Task Calendar() => TestModule("calendar"); + + [TestMethod, Priority(0)] + public Task Collections() => TestModule("collections"); + + [TestMethod, Priority(0)] + public Task Concurrent() => TestModule("concurrent"); + + [TestMethod, Priority(0)] + public Task Crypt() => TestModule("crypt"); + + [TestMethod, Priority(0)] + public Task Csv() => TestModule("csv"); + + [TestMethod, Priority(0)] + public Task CTypes() => TestModule("ctypes"); + + [TestMethod, Priority(0)] + public Task Curses() => TestModule("curses"); + + [TestMethod, Priority(0)] + public Task Dataclasses() => TestModule("dataclasses"); + + [TestMethod, Priority(0)] + public Task Datetime() => TestModule("datetime"); + + [TestMethod, Priority(0)] + public Task Dbm() => TestModule("dbm"); + + [TestMethod, Priority(0)] + public Task Distutils() => TestModule("distutils"); + + [TestMethod, Priority(0)] + public Task Email() => TestModule("email"); + + [TestMethod, Priority(0)] + public Task Encodings() => TestModule("encodings"); + + [TestMethod, Priority(0)] + public Task Enum() => TestModule("enum"); + + [TestMethod, Priority(0)] + public Task Filecmp() => TestModule("filecmp"); + + [TestMethod, Priority(0)] + public Task Fileinput() => TestModule("fileinput"); + + [TestMethod, Priority(0)] + public Task Fractions() => TestModule("fractions"); + + [TestMethod, Priority(0)] + public Task Ftplib() => TestModule("ftplib"); + + [TestMethod, Priority(0)] + public Task Functools() => TestModule("functools"); + + [TestMethod, Priority(0)] + public Task Genericpath() => TestModule("genericpath"); + + [TestMethod, Priority(0)] + public Task Glob() => TestModule("glob"); + + [TestMethod, Priority(0)] + public Task Gzip() => TestModule("gzip"); + + [TestMethod, Priority(0)] + public Task Hashlib() => TestModule("hashlib"); + + [TestMethod, Priority(0)] + public Task Heapq() => TestModule("heapq"); + + [TestMethod, Priority(0)] + public Task Html() => TestModule("html"); + + [TestMethod, Priority(0)] + public Task Http() => TestModule("http"); + + [TestMethod, Priority(0)] + public Task Importlib() => TestModule("importlib"); + + [TestMethod, Priority(0)] + public Task Imaplib() => TestModule("imaplib"); + + [TestMethod, Priority(0)] + public Task Imghdr() => TestModule("imghdr"); + + [TestMethod, Priority(0)] + public Task Inspect() => TestModule("inspect"); + + [TestMethod, Priority(0)] + public Task Io() => TestModule("io"); + + [TestMethod, Priority(0)] + public Task Json() => TestModule("json"); + + [TestMethod, Priority(0)] + public Task Logging() => TestModule("logging"); + + [TestMethod, Priority(0)] + public Task Lzma() => TestModule("lzma"); + + [TestMethod, Priority(0)] + public Task Mailbox() => TestModule("mailbox"); + + [TestMethod, Priority(0)] + public Task Multiprocessing() => TestModule("multiprocessing"); + + [TestMethod, Priority(0)] + public Task Numpy() => TestModule("numpy"); + + [TestMethod, Priority(0)] + public Task Os() => TestModule("os"); + + [TestMethod, Priority(0)] + public Task Pickle() => TestModule("pickle"); + + [TestMethod, Priority(0)] + public Task Pipes() => TestModule("pipes"); + + [TestMethod, Priority(0)] + public Task Pkgutil() => TestModule("pkgutil"); + + [TestMethod, Priority(0)] + [Ignore("https://github.com/microsoft/python-language-server/issues/1434")] + public Task Plistlib() => TestModule("plistlib"); + + [TestMethod, Priority(0)] + public Task Pstats() => TestModule("pstats"); + + [TestMethod, Priority(0)] + public Task Pydoc() => TestModule("pydoc"); + + [TestMethod, Priority(0)] + public Task Queue() => TestModule("queue"); + + [TestMethod, Priority(0)] + public Task Random() => TestModule("random"); + + [TestMethod, Priority(0)] + public Task Re() => TestModule("re"); + + [TestMethod, Priority(0)] + public Task Reprlib() => TestModule("reprlib"); + + [TestMethod, Priority(0)] + public Task Signal() => TestModule("signal"); + + [TestMethod, Priority(0)] + public Task Site() => TestModule("site"); + + [TestMethod, Priority(0)] + public Task Socket() => TestModule("socket"); + + [TestMethod, Priority(0)] + public Task Sqlite3() => TestModule("sqlite3"); + + [TestMethod, Priority(0)] + public Task Statistics() => TestModule("statistics"); + + [TestMethod, Priority(0)] + public Task String() => TestModule("string"); + + [TestMethod, Priority(0)] + public Task Ssl() => TestModule("ssl"); + + [TestMethod, Priority(0)] + public Task Sys() => TestModule("sys"); + + [TestMethod, Priority(0)] + public Task Tensorflow() => TestModule("tensorflow"); + + [TestMethod, Priority(0)] + public Task Time() => TestModule("time"); + + [TestMethod, Priority(0)] + public Task Threading() => TestModule("threading"); + + [TestMethod, Priority(0)] + public Task Tkinter() => TestModule("tkinter"); + + [TestMethod, Priority(0)] + public Task Token() => TestModule("token"); + + [TestMethod, Priority(0)] + public Task Trace() => TestModule("trace"); + + [TestMethod, Priority(0)] + public Task Types() => TestModule("types"); + + [TestMethod, Priority(0)] + public Task Unittest() => TestModule("unittest"); + + [TestMethod, Priority(0)] + public Task Urllib() => TestModule("urllib"); + + [TestMethod, Priority(0)] + public Task Uuid() => TestModule("uuid"); + + [TestMethod, Priority(0)] + public Task Weakref() => TestModule("weakref"); + + [TestMethod, Priority(0)] + public Task Xml() => TestModule("xml"); + + [TestMethod, Priority(0)] + public Task XmlRpc() => TestModule("xmlrpc"); + + [TestMethod, Priority(0)] + public Task Zipfile() => TestModule("zipfile"); + + [TestMethod, Priority(0)] + public async Task Requests() { + const string code = @" +import requests +x = requests.get('microsoft.com') +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var v = analysis.GlobalScope.Variables["requests"]; + v.Should().NotBeNull(); + if (v.Value.GetPythonType().ModuleType == ModuleType.Unresolved) { + Assert.Inconclusive("'requests' package is not installed."); + } + + var rq = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("requests"); + var model = ModuleModel.FromAnalysis(rq.Analysis, Services, AnalysisCachingLevel.Library); + + var u = model.UniqueId; + u.Should().Contain("(").And.EndWith(")"); + var open = u.IndexOf('('); + // Verify this looks like a version. + new Version(u.Substring(open + 1, u.IndexOf(')') - open - 1)); + + await CompareBaselineAndRestoreAsync(model, rq); + } + + private async Task TestModule(string name) { + var analysis = await GetAnalysisAsync($"import {name}", PythonVersions.Python37_x64); + + var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name); + if (m == null || m.ModuleType == ModuleType.Unresolved) { + Assert.Inconclusive($"Module {name} is not installed or otherwise could not be imported."); + return; + } + + var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); + model.Should().NotBeNull($"Module {name} is either not installed or cannot be cached"); + + await CompareBaselineAndRestoreAsync(model, m); + } + } +} diff --git a/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj b/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj new file mode 100644 index 000000000..514328cd5 --- /dev/null +++ b/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj @@ -0,0 +1,42 @@ + + + netcoreapp3.0 + Microsoft.Python.Analysis.Caching.Tests + Microsoft.Python.Analysis.Caching.Tests + + + + 1701;1702$(NoWarn) + 7.2 + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + + + + + diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs new file mode 100644 index 000000000..76b9ac803 --- /dev/null +++ b/src/Caching/Test/ReferencesTests.cs @@ -0,0 +1,119 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public class ReferencesTests : AnalysisCachingTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + private string BaselineFileName => GetBaselineFileName(TestContext.TestName); + + [TestMethod, Priority(0)] + public async Task MemberLocations() { + const string code = @" +x = 'str' + +def sum(a, b): + return a + b + +class B: + x: int + + class C: + def __init__(self): + pass + def methodC(self): + pass + + @property + def propertyB(self): + return 1 + + def methodB2(self): + return 2 +"; + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var json = ToJson(model); + Baseline.CompareToFile(BaselineFileName, json); + + using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + dbModule.Construct(model); + + var sum = dbModule.GetMember("sum") as IPythonFunctionType; + sum.Should().NotBeNull(); + sum.Definition.Span.Should().Be(4, 5, 4, 8); + + var b = dbModule.GetMember("B") as IPythonClassType; + b.Should().NotBeNull(); + b.Definition.Span.Should().Be(7, 7, 7, 8); + + var c = b.GetMember("C") as IPythonClassType; + c.Should().NotBeNull(); + c.Definition.Span.Should().Be(10, 11, 10, 12); + + var methodC = c.GetMember("methodC") as IPythonFunctionType; + methodC.Should().NotBeNull(); + methodC.Definition.Span.Should().Be(13, 13, 13, 20); + + var propertyB = b.GetMember("propertyB") as IPythonPropertyType; + propertyB.Should().NotBeNull(); + propertyB.Definition.Span.Should().Be(17, 9, 17, 18); + + var methodB2 = b.GetMember("methodB2") as IPythonFunctionType; + methodB2.Should().NotBeNull(); + methodB2.Definition.Span.Should().Be(20, 9, 20, 17); + } + } + + [TestMethod, Priority(0)] + public async Task Logging() { + const string code = @" +import logging +logging.critical() +"; + var analysis = await GetAnalysisAsync(code); + var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging"); + var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library); + + await CompareBaselineAndRestoreAsync(model, logging); + + using (var m = CreateDbModule(model, logging.FilePath)) { + var critical = m.GetMember("critical") as IPythonFunctionType; + critical.Should().NotBeNull(); + + var span = critical.Definition.Span; + span.Start.Line.Should().BeGreaterThan(1000); + (span.End.Column - span.Start.Column).Should().Be("critical".Length); + } + } + } +} diff --git a/src/Core/Impl/Extensions/StringExtensions.cs b/src/Core/Impl/Extensions/StringExtensions.cs index 878cfa2f8..be0b1f899 100644 --- a/src/Core/Impl/Extensions/StringExtensions.cs +++ b/src/Core/Impl/Extensions/StringExtensions.cs @@ -302,5 +302,16 @@ public static string NormalizeLineEndings(this string s, string lineEnding = nul lineEnding = lineEnding ?? Environment.NewLine; return string.Join(lineEnding, s.SplitLines()); } + + [DebuggerStepThrough] + public static int GetStableHash(this string s) { + unchecked { + var hash = 23; + foreach (var c in s) { + hash = hash * 31 + c; + } + return hash; + } + } } } diff --git a/src/Core/Impl/IO/DirectoryInfoProxy.cs b/src/Core/Impl/IO/DirectoryInfoProxy.cs index 8566f2267..1a6016dfb 100644 --- a/src/Core/Impl/IO/DirectoryInfoProxy.cs +++ b/src/Core/Impl/IO/DirectoryInfoProxy.cs @@ -53,7 +53,7 @@ public IEnumerable EnumerateFileSystemInfos(string searchPatter public IEnumerable EnumerateFileSystemInfos(string[] includePatterns, string[] excludePatterns) { var matcher = GetMatcher(includePatterns, excludePatterns); - PatternMatchingResult matchResult = SafeExecuteMatcher(matcher); + var matchResult = SafeExecuteMatcher(matcher); return matchResult.Files.Select((filePatternMatch) => { var path = PathUtils.NormalizePath(Path.Combine(_directoryInfo.FullName, filePatternMatch.Path)); return CreateFileSystemInfoProxy(new FileInfo(path)); diff --git a/src/Core/Impl/OS/IOSPlatform.cs b/src/Core/Impl/OS/IOSPlatform.cs index 0d14273ff..984441127 100644 --- a/src/Core/Impl/OS/IOSPlatform.cs +++ b/src/Core/Impl/OS/IOSPlatform.cs @@ -18,5 +18,6 @@ public interface IOSPlatform { bool IsWindows { get; } bool IsMac { get; } bool IsLinux { get; } + bool IsLittleEndian { get; } } } diff --git a/src/Core/Impl/OS/OSPlatform.cs b/src/Core/Impl/OS/OSPlatform.cs index c682ef5fb..09ddc752c 100644 --- a/src/Core/Impl/OS/OSPlatform.cs +++ b/src/Core/Impl/OS/OSPlatform.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Runtime.InteropServices; namespace Microsoft.Python.Core.OS { @@ -20,5 +21,6 @@ public sealed class OSPlatform : IOSPlatform { public bool IsWindows => RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows); public bool IsMac => RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX); public bool IsLinux => RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Linux); + public bool IsLittleEndian => BitConverter.IsLittleEndian; } } diff --git a/src/Core/Impl/OS/ProcessServices.cs b/src/Core/Impl/OS/ProcessServices.cs index 1f74efd40..e9a7c5b72 100644 --- a/src/Core/Impl/OS/ProcessServices.cs +++ b/src/Core/Impl/OS/ProcessServices.cs @@ -39,20 +39,20 @@ public IProcess Start(string path) { public async Task ExecuteAndCaptureOutputAsync(ProcessStartInfo startInfo, CancellationToken cancellationToken = default) { var output = string.Empty; - var process = Start(startInfo); + using (var process = Start(startInfo)) { - if (startInfo.RedirectStandardError && process is PlatformProcess p) { - p.Process.ErrorDataReceived += (s, e) => { }; - p.Process.BeginErrorReadLine(); - } + if (startInfo.RedirectStandardError && process is PlatformProcess p) { + p.Process.ErrorDataReceived += (s, e) => { }; + p.Process.BeginErrorReadLine(); + } - try { - output = await process.StandardOutput.ReadToEndAsync(); - await process.WaitForExitAsync(30000, cancellationToken); - } catch (IOException) { - } catch (OperationCanceledException) { } + try { + output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(30000, cancellationToken); + } catch (IOException) { } catch (OperationCanceledException) { } - return output; + return output; + } } } } diff --git a/src/Core/Impl/Text/ILocationConverter.cs b/src/Core/Impl/Text/ILocationConverter.cs new file mode 100644 index 000000000..162698200 --- /dev/null +++ b/src/Core/Impl/Text/ILocationConverter.cs @@ -0,0 +1,26 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Core.Text { + /// + /// Represents object that can convert linear span coordinates + /// to line/column and vise versa. + /// + public interface ILocationConverter { + SourceLocation IndexToLocation(int index); + int LocationToIndex(SourceLocation location); + } +} diff --git a/src/Core/Test/PriorityProducerConsumerTest.cs b/src/Core/Test/PriorityProducerConsumerTest.cs index 62c755ca6..a0a5132ac 100644 --- a/src/Core/Test/PriorityProducerConsumerTest.cs +++ b/src/Core/Test/PriorityProducerConsumerTest.cs @@ -24,95 +24,102 @@ namespace Microsoft.Python.Core.Tests { public class PriorityProducerConsumerTest { [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending() { - var ppc = new PriorityProducerConsumer(); - ppc.Produce(5); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer()) { + ppc.Produce(5); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Priority1() { - var ppc = new PriorityProducerConsumer(2); - ppc.Produce(5); - ppc.Produce(6, 1); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer(2)) { + ppc.Produce(5); + ppc.Produce(6, 1); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Priority2() { - var ppc = new PriorityProducerConsumer(2); - ppc.Produce(6, 1); - ppc.Produce(5); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + using (var ppc = new PriorityProducerConsumer(2)) { + ppc.Produce(6, 1); + ppc.Produce(5); + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates1() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5, 2); - ppc.Produce(6, 1); - ppc.Produce(5); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(5, consumerTask1.Result); - Assert.AreEqual(6, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5, 2); + ppc.Produce(6, 1); + ppc.Produce(5); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(5, consumerTask1.Result); + Assert.AreEqual(6, consumerTask2.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates2() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5); - ppc.Produce(6, 1); - ppc.Produce(5, 2); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(5, consumerTask1.Result); - Assert.AreEqual(6, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5); + ppc.Produce(6, 1); + ppc.Produce(5, 2); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(5, consumerTask1.Result); + Assert.AreEqual(6, consumerTask2.Result); + } } [TestMethod, Priority(0)] public void PriorityProducerConsumer_NoPending_Duplicates3() { - var ppc = new PriorityProducerConsumer(3, true); - ppc.Produce(5, 1); - ppc.Produce(6, 1); - ppc.Produce(5, 1); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - var consumerTask3 = ppc.ConsumeAsync(); - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); - Assert.AreEqual(6, consumerTask1.Result); - Assert.AreEqual(5, consumerTask2.Result); + using (var ppc = new PriorityProducerConsumer(3, true)) { + ppc.Produce(5, 1); + ppc.Produce(6, 1); + ppc.Produce(5, 1); + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + var consumerTask3 = ppc.ConsumeAsync(); + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask3.Status); + Assert.AreEqual(6, consumerTask1.Result); + Assert.AreEqual(5, consumerTask2.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending() { - var ppc = new PriorityProducerConsumer(); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + using (var ppc = new PriorityProducerConsumer()) { + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - ppc.Produce(5); - await consumerTask; + ppc.Produce(5); + await consumerTask; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] @@ -131,69 +138,72 @@ public async Task PriorityProducerConsumer_Pending_Dispose() { [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority1() { - var ppc = new PriorityProducerConsumer(2); - var consumerTask = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + using (var ppc = new PriorityProducerConsumer(2)) { + var consumerTask = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - ppc.Produce(5); - ppc.Produce(6, 1); - await consumerTask; + ppc.Produce(5); + ppc.Produce(6, 1); + await consumerTask; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, consumerTask.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, consumerTask.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority2() { - var ppc = new PriorityProducerConsumer(2); - var consumerTask1 = ppc.ConsumeAsync(); - var consumerTask2 = ppc.ConsumeAsync(); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); + using (var ppc = new PriorityProducerConsumer(2)) { + var consumerTask1 = ppc.ConsumeAsync(); + var consumerTask2 = ppc.ConsumeAsync(); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask1.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); - ppc.Produce(6, 1); - await consumerTask1; + ppc.Produce(6, 1); + await consumerTask1; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); - Assert.AreEqual(6, consumerTask1.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask1.Status); + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask2.Status); + Assert.AreEqual(6, consumerTask1.Result); - ppc.Produce(5); - await consumerTask2; + ppc.Produce(5); + await consumerTask2; - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); - Assert.AreEqual(5, consumerTask2.Result); + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask2.Status); + Assert.AreEqual(5, consumerTask2.Result); + } } [TestMethod, Priority(0)] public async Task PriorityProducerConsumer_Pending_Priority3() { - var ppc = new PriorityProducerConsumer(2); - var values = new int[3]; - var tcsConsumer = new TaskCompletionSource(); - var tcsProducer = new TaskCompletionSource(); - var consumerTask = Task.Run(async () => { - for (var i = 0; i < 3; i++) { - var task = ppc.ConsumeAsync(); - tcsConsumer.TrySetResult(true); - values[i] = await task; - await tcsProducer.Task; - } - }); - - Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); - - await tcsConsumer.Task; - ppc.Produce(5, 1); - ppc.Produce(6, 1); - ppc.Produce(7); - tcsProducer.SetResult(false); - - await consumerTask; - - Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); - Assert.AreEqual(5, values[0]); - Assert.AreEqual(7, values[1]); - Assert.AreEqual(6, values[2]); + using (var ppc = new PriorityProducerConsumer(2)) { + var values = new int[3]; + var tcsConsumer = new TaskCompletionSource(); + var tcsProducer = new TaskCompletionSource(); + var consumerTask = Task.Run(async () => { + for (var i = 0; i < 3; i++) { + var task = ppc.ConsumeAsync(); + tcsConsumer.TrySetResult(true); + values[i] = await task; + await tcsProducer.Task; + } + }); + + Assert.AreEqual(TaskStatus.WaitingForActivation, consumerTask.Status); + + await tcsConsumer.Task; + ppc.Produce(5, 1); + ppc.Produce(6, 1); + ppc.Produce(7); + tcsProducer.SetResult(false); + + await consumerTask; + + Assert.AreEqual(TaskStatus.RanToCompletion, consumerTask.Status); + Assert.AreEqual(5, values[0]); + Assert.AreEqual(7, values[1]); + Assert.AreEqual(6, values[2]); + } } } } diff --git a/src/LanguageServer/Impl/Completion/CompletionItemSource.cs b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs index 60e43e7a2..8e1fbc6ee 100644 --- a/src/LanguageServer/Impl/Completion/CompletionItemSource.cs +++ b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs @@ -72,7 +72,7 @@ public static CompletionItemEx CreateCompletionItem(string text, CompletionItemK private static CompletionItemKind ToCompletionItemKind(PythonMemberType memberType) { switch (memberType) { - case PythonMemberType.Unknown: return CompletionItemKind.None; + case PythonMemberType.Unknown: return CompletionItemKind.Text; case PythonMemberType.Class: return CompletionItemKind.Class; case PythonMemberType.Instance: return CompletionItemKind.Value; case PythonMemberType.Function: return CompletionItemKind.Function; @@ -83,7 +83,7 @@ private static CompletionItemKind ToCompletionItemKind(PythonMemberType memberTy case PythonMemberType.Variable: return CompletionItemKind.Variable; case PythonMemberType.Generic: return CompletionItemKind.TypeParameter; } - return CompletionItemKind.None; + return CompletionItemKind.Text; } } } diff --git a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs index a199acae7..5e9ff8d33 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs @@ -13,15 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; -using System.Linq; using Microsoft.Python.Analysis; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.Python.LanguageServer.Completion { internal static class ExpressionCompletion { diff --git a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs index f5fcc7ad0..046488540 100644 --- a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs +++ b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs @@ -87,7 +87,7 @@ private int NextBlockIndent private string CurrentLineWithinBlock => CurrentLine.Substring(_blockIndent); private DocstringConverter(string input) { - _builder = new StringBuilder(input.Length); + _builder = new StringBuilder(input?.Length ?? 0); _state = ParseText; _lines = SplitDocstring(input); } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 7b5f66f03..01c643ea9 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -22,6 +22,7 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; @@ -139,15 +140,15 @@ public async Task InitializedAsync(InitializedParams @params, CancellationToken Version.TryParse(initializationOptions?.interpreter.properties?.Version, out var version); - var configuration = new InterpreterConfiguration(null, null, + var configuration = new InterpreterConfiguration( interpreterPath: initializationOptions?.interpreter.properties?.InterpreterPath, version: version ); + _services.AddService(new ModuleDatabase(_services)); var typeshedPath = initializationOptions?.typeStubSearchPaths.FirstOrDefault(); userConfiguredPaths = userConfiguredPaths ?? initializationOptions?.searchPaths; _interpreter = await PythonInterpreter.CreateAsync(configuration, Root, _services, typeshedPath, userConfiguredPaths.ToImmutableArray(), cancellationToken); - _services.AddService(_interpreter); _log?.Log(TraceEventType.Information, string.IsNullOrEmpty(_interpreter.Configuration.InterpreterPath) diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 82765a0fa..6a21b2f1c 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -16,7 +16,7 @@ using System; using System.Collections.Generic; -using System.IO; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -33,6 +33,7 @@ using Microsoft.Python.LanguageServer.SearchPaths; using Newtonsoft.Json.Linq; using StreamJsonRpc; +using DiagnosticSource = Microsoft.Python.Analysis.Diagnostics.DiagnosticSource; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class LanguageServer { @@ -41,6 +42,8 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell using (await _prioritizer.ConfigurationPriorityAsync(cancellationToken)) { var settings = new LanguageServerSettings(); + // https://github.com/microsoft/python-language-server/issues/915 + // If token or settings are missing, assume defaults. var rootSection = token?["settings"]; var pythonSection = rootSection?["python"]; if (pythonSection == null) { @@ -87,6 +90,9 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin var optionsProvider = _services.GetService(); optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); + optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); + + _logger?.Log(TraceEventType.Information, Resources.AnalysisCacheLevel.FormatInvariant(optionsProvider.Options.AnalysisCachingLevel)); } internal static void HandleLintingOnOff(IServiceContainer services, bool linterEnabled) { @@ -173,5 +179,13 @@ private ImmutableArray GetUserConfiguredPaths(JToken pythonSection) { return ImmutableArray.Empty; } + + private AnalysisCachingLevel GetAnalysisCachingLevel(JToken analysisKey) { + var s = GetSetting(analysisKey, "cachingLevel", "None"); + if (s.EqualsIgnoreCase("System")) { + return AnalysisCachingLevel.System; + } + return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; + } } } diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index 1ee0e8a67..ca455737f 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -43,6 +43,7 @@ + diff --git a/src/LanguageServer/Impl/Protocol/Enums.cs b/src/LanguageServer/Impl/Protocol/Enums.cs index 1bc87af14..08cfeef7a 100644 --- a/src/LanguageServer/Impl/Protocol/Enums.cs +++ b/src/LanguageServer/Impl/Protocol/Enums.cs @@ -146,7 +146,9 @@ public enum InsertTextFormat { } public enum CompletionItemKind { - None = 0, + // Do not return 0 or anything not in this list. + // See https://microsoft.github.io/language-server-protocol/specification + // VS Code converts values outside of the LSP range into Text. Text = 1, Method = 2, Function = 3, diff --git a/src/LanguageServer/Impl/Resources.Designer.cs b/src/LanguageServer/Impl/Resources.Designer.cs index 96723de5d..b413d0dde 100644 --- a/src/LanguageServer/Impl/Resources.Designer.cs +++ b/src/LanguageServer/Impl/Resources.Designer.cs @@ -60,6 +60,15 @@ internal Resources() { } } + /// + /// Looks up a localized string similar to Analysis caching mode: {0}.. + /// + internal static string AnalysisCacheLevel { + get { + return ResourceManager.GetString("AnalysisCacheLevel", resourceCulture); + } + } + /// /// Looks up a localized string similar to Code analysis is in progress.... /// @@ -132,6 +141,15 @@ internal static string LineFormatter_UnmatchedToken { } } + /// + /// Looks up a localized string similar to Python interpreter version not specified. Assuming latests 3.x.. + /// + internal static string PythonInterpreterVersionNotSpecified { + get { + return ResourceManager.GetString("PythonInterpreterVersionNotSpecified", resourceCulture); + } + } + /// /// Looks up a localized string similar to Reloading modules... . /// diff --git a/src/LanguageServer/Impl/Resources.resx b/src/LanguageServer/Impl/Resources.resx index 024751798..e060c233d 100644 --- a/src/LanguageServer/Impl/Resources.resx +++ b/src/LanguageServer/Impl/Resources.resx @@ -117,6 +117,9 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Analysis caching mode: {0}. + Code analysis is in progress... @@ -141,6 +144,9 @@ Unmatched token '{0}' on line {1}; line formatting may not be accurate. + + Python interpreter version not specified. Assuming latests 3.x. + Reloading modules... diff --git a/src/LanguageServer/Impl/Sources/DefinitionSource.cs b/src/LanguageServer/Impl/Sources/DefinitionSource.cs index 253aa5325..56f1e8a1a 100644 --- a/src/LanguageServer/Impl/Sources/DefinitionSource.cs +++ b/src/LanguageServer/Impl/Sources/DefinitionSource.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Completion; using Microsoft.Python.LanguageServer.Protocol; @@ -112,49 +113,68 @@ public Reference FindDefinition(IDocumentAnalysis analysis, SourceLocation locat private Reference HandleFromImport(IDocumentAnalysis analysis, SourceLocation location, FromImportStatement statement, Node expr, out ILocatedMember definingMember) { definingMember = null; - // Are in the dotted name? + var mres = analysis.Document.Interpreter.ModuleResolution; + var imports = mres.CurrentPathResolver.FindImports(analysis.Document.FilePath, statement); + IPythonModule module = null; + switch (imports) { + case ModuleImport moduleImport: + module = mres.GetImportedModule(moduleImport.FullName); + break; + case ImplicitPackageImport packageImport: + module = mres.GetImportedModule(packageImport.FullName); + break; + } + + // Are we in the module name (i.e. A in 'from A import B')? var locationIndex = location.ToIndex(analysis.Ast); if (statement.Root.StartIndex <= locationIndex && locationIndex <= statement.Root.EndIndex) { - var mres = analysis.Document.Interpreter.ModuleResolution; - var imports = mres.CurrentPathResolver.FindImports(analysis.Document.FilePath, statement); - IPythonModule module = null; - switch (imports) { - case ModuleImport moduleImport: - module = mres.GetImportedModule(moduleImport.FullName); - break; - case ImplicitPackageImport packageImport: - module = mres.GetImportedModule(packageImport.FullName); - break; - } - definingMember = module; return module != null ? new Reference { range = default, uri = CanNavigateToModule(module) ? module.Uri : null } : null; } - // We are in what/as part - var nex = expr as NameExpression; - var name = nex?.Name; + if (module == null) { + return null; + } + + // Are we in the member name part (ie. B in 'from A import B')? + // Handle 'from A import B' similar to 'import A.B' + var partReference = FindModulePartReference(statement.Names, expr, module, out definingMember); + if (partReference != null) { + return partReference; + } + + // Are we in 'as' names? + var name = (expr as NameExpression)?.Name; if (string.IsNullOrEmpty(name)) { return null; } - // From X import A - var value = analysis.ExpressionEvaluator.GetValueFromExpression(nex); - if (value.IsUnknown()) { - // From X import A as B - var index = statement.Names.IndexOf(x => x?.Name == name); - if (index >= 0 && index < statement.AsNames.Count) { - value = analysis.ExpressionEvaluator.GetValueFromExpression(statement.AsNames[index]); + var asName = statement.AsNames.FirstOrDefault(x => x?.Name == name); + if (asName != null) { + var value = analysis.ExpressionEvaluator.GetValueFromExpression(asName); + if (!value.IsUnknown()) { + definingMember = value as ILocatedMember; + return FromMember(definingMember); } } - if (!value.IsUnknown()) { - definingMember = value as ILocatedMember; - return FromMember(definingMember); - } + return null; + } + private static Reference FindModulePartReference(ImmutableArray names, Node expr, IPythonModule module, out ILocatedMember definingMember) { + definingMember = null; + var part = names.FirstOrDefault(x => x.IndexSpan.Start <= expr.StartIndex && x.IndexSpan.Start <= expr.EndIndex); + if (part != null) { + var definition = module.Analysis.GlobalScope.Variables[part.Name]?.Definition; + if (definition != null) { + return new Reference { + range = definition.Span, + uri = CanNavigateToModule(module) ? module.Uri : null + }; + } + } return null; } @@ -166,20 +186,15 @@ private Reference HandleImport(IDocumentAnalysis analysis, ImportStatement state return null; } - var index = statement.Names.IndexOf(x => x?.MakeString() == name); - if (index < 0) { - return null; - } - - var module = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name); - if (module != null) { - definingMember = module; - return new Reference { range = default, uri = CanNavigateToModule(module) ? module.Uri : null }; + var reference = FindModuleNamePartReference(analysis, statement.Names, expr, out definingMember); + if(reference != null) { + return reference; } // Import A as B - if (index >= 0 && index < statement.AsNames.Count) { - var value = analysis.ExpressionEvaluator.GetValueFromExpression(statement.AsNames[index]); + var asName = statement.AsNames.FirstOrDefault(n => n.IndexSpan.Start <= expr.StartIndex && n.IndexSpan.Start <= expr.EndIndex); + if (asName != null) { + var value = analysis.ExpressionEvaluator.GetValueFromExpression(asName); if (!value.IsUnknown()) { definingMember = value as ILocatedMember; return FromMember(definingMember); @@ -188,12 +203,51 @@ private Reference HandleImport(IDocumentAnalysis analysis, ImportStatement state return null; } + /// + /// Given dotted name located reference to the part of the name. For example, given + /// 'os.path' and the name expression 'path' locates definition of 'path' part of 'os' module. + /// + private static Reference FindModuleNamePartReference(IDocumentAnalysis analysis, ImmutableArray dottedName, Node expr, out ILocatedMember definingMember) { + definingMember = null; + var moduleName = dottedName.FirstOrDefault(x => x.IndexSpan.Start <= expr.StartIndex && x.IndexSpan.Start <= expr.EndIndex); + if (moduleName == null) { + return null; + } + + var module = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(moduleName.Names.First().Name); + foreach (var member in moduleName.Names.Skip(1)) { + if (module == null) { + return null; + } + + if (member.StartIndex >= expr.EndIndex) { + break; + } + + if (member.StartIndex <= expr.EndIndex && member.EndIndex <= expr.EndIndex) { + var definition = module.Analysis.GlobalScope.Variables[member.Name]?.Definition; + if (definition != null) { + return new Reference { + range = definition.Span, + uri = CanNavigateToModule(module) ? module.Uri : null + }; + } + } + module = module.GetMember(member.Name) as IPythonModule; + } + + if (module != null) { + definingMember = module; + return new Reference { range = default, uri = CanNavigateToModule(module) ? module.Uri : null }; + } + return null; + } private Reference TryFromVariable(string name, IDocumentAnalysis analysis, SourceLocation location, Node statement, out ILocatedMember definingMember) { definingMember = null; var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope, LookupOptions.All); - if (m == null || !(scope.Variables[name] is IVariable v)) { + if (m == null || scope.Module.ModuleType == ModuleType.Builtins || !(scope.Variables[name] is IVariable v)) { return null; } @@ -223,10 +277,10 @@ private Reference FromMemberExpression(MemberExpression mex, IDocumentAnalysis a var type = target?.GetPythonType(); switch (type) { - case IPythonModule m when m.Analysis.GlobalScope != null: + case IPythonModule m when m.GlobalScope != null: // Module GetMember returns module variable value while we // want the variable itself since we want to know its location. - var v1 = m.Analysis.GlobalScope.Variables[mex.Name]; + var v1 = m.GlobalScope.Variables[mex.Name]; if (v1 != null) { definingMember = v1; return FromMember(v1); @@ -278,12 +332,16 @@ private bool CanNavigateToModule(Uri uri) { } var rdt = _services.GetService(); var doc = rdt.GetDocument(uri); - return CanNavigateToModule(doc); + // Allow navigation to modules not in RDT - most probably + // it is a module that was restored from database. + return doc == null || CanNavigateToModule(doc); } private static bool CanNavigateToModule(IPythonModule m) => m?.ModuleType == ModuleType.User || + m?.ModuleType == ModuleType.Stub || m?.ModuleType == ModuleType.Package || - m?.ModuleType == ModuleType.Library; + m?.ModuleType == ModuleType.Library || + m?.ModuleType == ModuleType.Specialized; } } diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs index 99f670fe8..ca846f513 100644 --- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -58,7 +58,7 @@ public async Task FindAllReferencesAsync(Uri uri, SourceLocation lo // If it is an implicitly declared variable, such as function or a class // then the location is invalid and the module is null. Use current module. - var declaringModule = rootDefinition.DeclaringModule ?? analysis.Document; + var declaringModule = rootDefinition?.DeclaringModule ?? analysis.Document; if (!string.IsNullOrEmpty(name) && (declaringModule.ModuleType == ModuleType.User || options == ReferenceSearchOptions.All)) { return await FindAllReferencesAsync(name, declaringModule, rootDefinition, location, definitionSource, cancellationToken); } @@ -100,7 +100,7 @@ private async Task FindAllReferencesAsync(string name, IPythonModul return new List<(Uri, long)>(); } - var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths.ToArray(); + var interpreterPaths = interpreter.ModuleResolution.InterpreterPaths; var files = new List<(Uri, long)>(); foreach (var filePath in fs.GetFiles(root, "*.py", SearchOption.AllDirectories).Select(Path.GetFullPath)) { diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index bb2cdd82a..625eba996 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -412,7 +412,7 @@ public async Task MarkupKindValid() { var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); var result = cs.GetCompletions(analysis, new SourceLocation(2, 5)); - result.Completions?.Select(i => i.documentation.kind) + result.Completions?.Select(i => i.documentation?.kind).ExcludeDefault() .Should().NotBeEmpty().And.BeSubsetOf(new[] { MarkupKind.PlainText, MarkupKind.Markdown }); } diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index e5369c56c..082ea06c2 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -464,5 +463,114 @@ def foo(self): reference.range.Should().Be(2, 8, 2, 16); reference.uri.AbsolutePath.Should().Contain("bar.py"); } + + [TestMethod, Priority(0)] + public async Task NamedTuple() { + const string code = @" +from typing import NamedTuple + +Point = NamedTuple('Point', ['x', 'y']) + +def f(a, b): + return Point(a, b) + +pt = Point(1, 2) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(7, 14), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(3, 0, 3, 5); + } + + [TestMethod, Priority(0)] + public async Task ModulePartsNavigation() { + const string code = @" +import os.path +from os import path as os_path +print(os.path.basename('a/b/c')) +print(os_path.basename('a/b/c')) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(2, 9), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(0, 0, 0, 0); + reference.uri.AbsolutePath.Should().Contain("os.py"); + + reference = ds.FindDefinition(analysis, new SourceLocation(2, 13), out _); + reference.Should().NotBeNull(); + var line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + line.Should().EndWith("as path"); + line.Substring(reference.range.start.character).Should().Be("path"); + + reference = ds.FindDefinition(analysis, new SourceLocation(3, 7), out _); + reference.Should().NotBeNull(); + reference.range.Should().Be(0, 0, 0, 0); + reference.uri.AbsolutePath.Should().Contain("os.py"); + + reference = ds.FindDefinition(analysis, new SourceLocation(3, 17), out _); + reference.Should().NotBeNull(); + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + line.Should().EndWith("as path"); + line.Substring(reference.range.start.character).Should().Be("path"); + + reference = ds.FindDefinition(analysis, new SourceLocation(3, 27), out _); + reference.Should().NotBeNull(); + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + line.Should().EndWith("as path"); + line.Substring(reference.range.start.character).Should().Be("path"); + + reference = ds.FindDefinition(analysis, new SourceLocation(4, 12), out _); + reference.Should().NotBeNull(); + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + line.Should().EndWith("as path"); + line.Substring(reference.range.start.character).Should().Be("path"); + + reference = ds.FindDefinition(analysis, new SourceLocation(5, 12), out _); + reference.Should().NotBeNull(); + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + line.Should().EndWith("as path"); + line.Substring(reference.range.start.character).Should().Be("path"); + } + + [TestMethod, Priority(0)] + public async Task Unittest() { + const string code = @" +from unittest import TestCase + +class MyTestCase(TestCase): + def test_example(self): + with self.assertRaises(ZeroDivisionError): + value = 1 / 0 + self.assertNotEqual(value, 1) +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(6, 24), out _); + reference.Should().NotBeNull(); + reference.range.start.line.Should().BeGreaterThan(0); + reference.uri.AbsolutePath.Should().Contain("case.py"); + reference.uri.AbsolutePath.Should().NotContain("pyi"); + } + + [TestMethod, Priority(0)] + public async Task DateTimeProperty() { + const string code = @" +import datetime +x = datetime.datetime.day +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var ds = new DefinitionSource(Services); + + var reference = ds.FindDefinition(analysis, new SourceLocation(3, 15), out _); + reference.Should().NotBeNull(); + reference.range.start.line.Should().BeGreaterThan(0); + reference.uri.AbsolutePath.Should().Contain("datetime.py"); + reference.uri.AbsolutePath.Should().NotContain("pyi"); + } } } diff --git a/src/LanguageServer/Test/HoverTests.cs b/src/LanguageServer/Test/HoverTests.cs index 50d42aa2e..2d8623d32 100644 --- a/src/LanguageServer/Test/HoverTests.cs +++ b/src/LanguageServer/Test/HoverTests.cs @@ -89,7 +89,7 @@ public async Task HoverSpanCheck(bool is3x) { import datetime datetime.datetime.now().day "; - var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); var hs = new HoverSource(new PlainTextDocumentationSource()); AssertHover(hs, analysis, new SourceLocation(3, 2), "module datetime*", new SourceSpan(3, 1, 3, 9)); diff --git a/src/LanguageServer/Test/IndexManagerTests.cs b/src/LanguageServer/Test/IndexManagerTests.cs index d71ae58e8..00071495d 100644 --- a/src/LanguageServer/Test/IndexManagerTests.cs +++ b/src/LanguageServer/Test/IndexManagerTests.cs @@ -52,91 +52,96 @@ public void Cleanup() { [TestMethod, Priority(0)] public async Task AddsRootDirectoryAsync() { - var context = new IndexTestContext(this); - context.FileWithXVarInRootDir(); - context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); + using (var context = new IndexTestContext(this)) { + context.FileWithXVarInRootDir(); + context.AddFileToRoot($"{_rootPath}\foo.py", MakeStream("y = 1")); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(2); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } } [TestMethod, Priority(0)] public async Task IgnoresNonPythonFiles() { - var context = new IndexTestContext(this); - - var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); - context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); + using (var context = new IndexTestContext(this)) { + var nonPythonTestFileInfo = MakeFileInfoProxy($"{_rootPath}/bla.txt"); + context.AddFileInfoToRootTestFS(nonPythonTestFileInfo); - IIndexManager indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + context.FileSystem.DidNotReceive().FileExists(nonPythonTestFileInfo.FullName); + } } [TestMethod, Priority(0)] public async Task CanOpenFiles() { - string nonRootPath = "C:/nonRoot"; - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); - IDocument doc = DocumentWithAst("x = 1"); + var nonRootPath = "C:/nonRoot"; + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy($"{nonRootPath}/bla.py"); + var doc = DocumentWithAst("x = 1"); - IIndexManager indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, doc); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task UpdateFilesOnWorkspaceIndexesLatestAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ReIndexFile(pythonTestFilePath, DocumentWithAst("y = 1")); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("y"); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("y"); + } } [TestMethod, Priority(0)] public async Task CloseNonWorkspaceFilesRemovesFromIndexAsync() { - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - await SymbolIndexShouldBeEmpty(indexManager); + await SymbolIndexShouldBeEmpty(indexManager); + } } [TestMethod, Priority(0)] public async Task CloseWorkspaceFilesReUpdatesIndexAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); - - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); - // It Needs to remake the stream for the file, previous one is closed - context.FileSystem.FileExists(pythonTestFilePath).Returns(true); - context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); - indexManager.ProcessClosedFile(pythonTestFilePath); - - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); + + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("r = 1")); + // It Needs to remake the stream for the file, previous one is closed + context.FileSystem.FileExists(pythonTestFilePath).Returns(true); + context.SetFileOpen(pythonTestFilePath, MakeStream("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFilePath).Returns(true); + indexManager.ProcessClosedFile(pythonTestFilePath); + + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] @@ -144,92 +149,97 @@ public async Task ProcessFileIfIndexedAfterCloseIgnoresUpdateAsync() { // If events get to index manager in the order: [open, close, update] // it should not reindex file - var context = new IndexTestContext(this); - var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); + using (var context = new IndexTestContext(this)) { + var pythonTestFileInfo = MakeFileInfoProxy("C:/nonRoot/bla.py"); - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + indexManager.ProcessClosedFile(pythonTestFileInfo.FullName); - context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); - indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); + context.FileSystem.IsPathUnderRoot(_rootPath, pythonTestFileInfo.FullName).Returns(false); + indexManager.ReIndexFile(pythonTestFileInfo.FullName, DocumentWithAst("x = 1")); - await SymbolIndexShouldBeEmpty(indexManager); + await SymbolIndexShouldBeEmpty(indexManager); + } } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsAddsRootDirectory() { - var context = new IndexTestContext(this); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); - - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task WorkspaceSymbolsLimited() { - var context = new IndexTestContext(this); + using (var context = new IndexTestContext(this)) { + for (var fileNumber = 0; fileNumber < 10; fileNumber++) { + context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); + } - for (int fileNumber = 0; fileNumber < 10; fileNumber++) { - context.AddFileToRoot($"{_rootPath}\bla{fileNumber}.py", MakeStream($"x{fileNumber} = 1")); - } - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - const int amountOfSymbols = 3; + const int amountOfSymbols = 3; - var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); - symbols.Should().HaveCount(amountOfSymbols); + var symbols = await indexManager.WorkspaceSymbolsAsync("", amountOfSymbols); + symbols.Should().HaveCount(amountOfSymbols); + } } [TestMethod, Priority(0)] public async Task HierarchicalDocumentSymbolsAsync() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); - SymbolsShouldBeOnlyX(symbols); + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + SymbolsShouldBeOnlyX(symbols); + } } [TestMethod, Priority(0)] public async Task LatestVersionASTVersionIsIndexed() { - var context = new IndexTestContext(this); - var pythonTestFilePath = context.FileWithXVarInRootDir(); - - var indexManager = context.GetDefaultIndexManager(); - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); - indexManager.ProcessClosedFile(pythonTestFilePath); - indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); - - var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("z"); + using (var context = new IndexTestContext(this)) { + var pythonTestFilePath = context.FileWithXVarInRootDir(); + + var indexManager = context.GetDefaultIndexManager(); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("y = 1")); + indexManager.ProcessClosedFile(pythonTestFilePath); + indexManager.ProcessNewFile(pythonTestFilePath, DocumentWithAst("z = 1")); + + var symbols = await indexManager.HierarchicalDocumentSymbolsAsync(pythonTestFilePath); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("z"); + } } [TestMethod, Priority(0)] public async Task AddFilesToPendingChanges() { - var context = new IndexTestContext(this); - var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); - var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); + using (var context = new IndexTestContext(this)) { + var f1 = context.AddFileToRoot($"{_rootPath}/fileA.py", MakeStream("")); + var f2 = context.AddFileToRoot($"{_rootPath}/fileB.py", MakeStream("")); - var indexManager = context.GetDefaultIndexManager(); - await indexManager.IndexWorkspace(); + var indexManager = context.GetDefaultIndexManager(); + await indexManager.IndexWorkspace(); - indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); - indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); + indexManager.AddPendingDoc(DocumentWithAst("y = 1", f1)); + indexManager.AddPendingDoc(DocumentWithAst("x = 1", f2)); - context.SetIdleEvent(Raise.Event()); + context.SetIdleEvent(Raise.Event()); - var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); - symbols.Should().HaveCount(2); + var symbols = await indexManager.WorkspaceSymbolsAsync("", maxSymbolsCount); + symbols.Should().HaveCount(2); + } } private static void SymbolsShouldBeOnlyX(IEnumerable symbols) { @@ -249,8 +259,8 @@ private class IndexTestContext : IDisposable { private readonly List _rootFileList = new List(); private readonly IIdleTimeService _idleTimeService = Substitute.For(); private readonly PythonLanguageVersion _pythonLanguageVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + private readonly IndexManagerTests _tests; private IIndexManager _indexM; - private IndexManagerTests _tests; public IndexTestContext(IndexManagerTests tests) { _tests = tests; @@ -271,9 +281,7 @@ public void AddFileInfoToRootTestFS(FileInfoProxy fileInfo) { FileSystem.FileExists(fileInfo.FullName).Returns(true); } - public string FileWithXVarInRootDir() { - return AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); - } + public string FileWithXVarInRootDir() => AddFileToRoot($"{_rootPath}\bla.py", _tests.MakeStream("x = 1")); public IIndexManager GetDefaultIndexManager() { _indexM = new IndexManager(FileSystem, _pythonLanguageVersion, @@ -301,19 +309,16 @@ public void SetIdleEvent(EventHandler handler) { private void SetupRootDir() { var directoryInfo = Substitute.For(); directoryInfo.Match("", new string[] { }, new string[] { }).ReturnsForAnyArgs(callInfo => { - string path = callInfo.ArgAt(0); + var path = callInfo.ArgAt(0); return _rootFileList - .Where(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) - .Count() > 0; + .Count(fsInfo => PathEqualityComparer.Instance.Equals(fsInfo.FullName, path)) > 0; }); // Doesn't work without 'forAnyArgs' directoryInfo.EnumerateFileSystemInfos(new string[] { }, new string[] { }).ReturnsForAnyArgs(_rootFileList); FileSystem.GetDirectoryInfo(_rootPath).Returns(directoryInfo); } - public void Dispose() { - _indexM?.Dispose(); - } + public void Dispose() => _indexM?.Dispose(); public void SetFileOpen(string pythonTestFilePath, Func returnFunc) { FileSystem.FileOpen(pythonTestFilePath, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(returnFunc); @@ -326,7 +331,7 @@ internal void SetFileOpen(string path, Stream stream) { private IDocument DocumentWithAst(string testCode, string filePath = null) { filePath = filePath ?? $"{_rootPath}/{testCode}.py"; - IDocument doc = Substitute.For(); + var doc = Substitute.For(); doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); doc.Uri.Returns(new Uri(filePath)); return doc; @@ -338,13 +343,11 @@ private async Task SymbolIndexShouldBeEmpty(IIndexManager indexManager) { } public PythonAst MakeAst(string testCode) { - PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + var latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); } - public Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); - } + public Stream MakeStream(string str) => new MemoryStream(Encoding.UTF8.GetBytes(str)); public FileInfoProxy MakeFileInfoProxy(string filePath) => new FileInfoProxy(new FileInfo(filePath)); diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index ecd86654d..32e66ec26 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -56,15 +56,15 @@ public async Task ParseVariableInFileAsync() { using (var fileStream = MakeStream("x = 1")) { SetFileOpen(_fileSystem, testFilePath, fileStream); - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - var ast = await indexParser.ParseAsync(testFilePath); - - var symbols = GetIndexSymbols(ast); - symbols.Should().HaveCount(1); - symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); - symbols.First().Name.Should().BeEquivalentTo("x"); + using (IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { + var ast = await indexParser.ParseAsync(testFilePath); + + var symbols = GetIndexSymbols(ast); + symbols.Should().HaveCount(1); + symbols.First().Kind.Should().BeEquivalentTo(SymbolKind.Variable); + symbols.First().Name.Should().BeEquivalentTo("x"); + } } - } private IReadOnlyList GetIndexSymbols(PythonAst ast) { @@ -81,8 +81,9 @@ public async Task ParseFileThatStopsExisting() { _fileSystem.FileExists(testFilePath).Returns(true); SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - var ast = await indexParser.ParseAsync(testFilePath); + using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { + await indexParser.ParseAsync(testFilePath); + } } [TestMethod, Priority(0)] @@ -94,26 +95,25 @@ public void CancelParsingAsync() { SetFileOpen(_fileSystem, testFilePath, fileStream); } - IIndexParser indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.Cancel(); + using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) + using (var cancellationTokenSource = new CancellationTokenSource()) { + cancellationTokenSource.Cancel(); - Func parse = async () => { - await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); - }; - parse.Should().Throw(); + Func parse = async () => { + await indexParser.ParseAsync(testFilePath, cancellationTokenSource.Token); + }; + parse.Should().Throw(); + } } private void SetFileOpen(IFileSystem fileSystem, string path, Stream stream) { fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(stream); } - private void SetFileOpen(IFileSystem fileSystem, string path, Func p) { - fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); - } + private void SetFileOpen(IFileSystem fileSystem, string path, Func p) + => fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read).Returns(p); - private Stream MakeStream(string str) { - return new MemoryStream(Encoding.UTF8.GetBytes(str)); - } + private Stream MakeStream(string str) + => new MemoryStream(Encoding.UTF8.GetBytes(str)); } } diff --git a/src/LanguageServer/Test/SymbolIndexTests.cs b/src/LanguageServer/Test/SymbolIndexTests.cs index 0a91ba2b0..861eb1cec 100644 --- a/src/LanguageServer/Test/SymbolIndexTests.cs +++ b/src/LanguageServer/Test/SymbolIndexTests.cs @@ -27,6 +27,7 @@ using Microsoft.Python.Parsing.Ast; using Microsoft.Python.Parsing.Tests; using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.Python.UnitTests.Core.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using NSubstitute; using TestUtilities; @@ -51,14 +52,15 @@ public void TestCleanup() { [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } } private static ISymbolIndex MakeSymbolIndex() { @@ -67,124 +69,130 @@ private static ISymbolIndex MakeSymbolIndex() { [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentUpdate() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); + index.Add(path, DocumentWithAst("x = 1")); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); - index.Add(path, DocumentWithAst("y = 1")); + index.Add(path, DocumentWithAst("y = 1")); - symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), - }); + symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)) + }); + } } [TestMethod, Priority(0)] public async Task IndexHierarchicalDocumentNotFoundAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - var symbols = await index.HierarchicalDocumentSymbolsAsync(path); - symbols.Should().BeEmpty(); + var symbols = await index.HierarchicalDocumentSymbolsAsync(path); + symbols.Should().BeEmpty(); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsFlattenAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst(code)); - index.Add(path, DocumentWithAst(code)); - - var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), - new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, path, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo") + }); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsFilteredAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst(code)); + index.Add(path, DocumentWithAst(code)); - var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("x", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsNotFoundAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - - var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); - symbols.Should().BeEmpty(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + var symbols = await index.WorkspaceSymbolsAsync("", maxSymbols); + symbols.Should().BeEmpty(); + } } [TestMethod, Priority(0)] public async Task IndexWorkspaceSymbolsCaseInsensitiveAsync() { - var code = @"class Foo(object): + const string code = @"class Foo(object): def foo(self, x): ..."; - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + index.Add(path, DocumentWithAst(code)); - index.Add(path, DocumentWithAst(code)); - - var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), - new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), - }); + var symbols = await index.WorkspaceSymbolsAsync("foo", maxSymbols); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, path, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, path, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } } [TestMethod, Priority(0)] public void MarkAsPendingWaitsForUpdates() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); - - index.Add(path, DocumentWithAst("x = 1")); - index.MarkAsPending(path); - var cts = new CancellationTokenSource(); - var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); - t.IsCompleted.Should().BeFalse(); - cts.Cancel(); - Func cancelled = async () => { - await t; - }; - cancelled.Should().Throw(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); + + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + using (var cts = new CancellationTokenSource()) { + var t = index.HierarchicalDocumentSymbolsAsync(path, cts.Token); + t.Should().NotBeCompleted(); + cts.Cancel(); + Func cancelled = async () => { + await t; + }; + cancelled.Should().Throw(); + } + } } [TestMethod, Priority(0)] public async Task SymbolsAfterPendingWaitsForUpdateAsync() { - ISymbolIndex index = MakeSymbolIndex(); - var path = TestData.GetDefaultModulePath(); + using (var index = MakeSymbolIndex()) { + var path = TestData.GetDefaultModulePath(); - index.Add(path, DocumentWithAst("x = 1")); - index.MarkAsPending(path); - var t = index.WorkspaceSymbolsAsync("", maxSymbols); - index.ReIndex(path, DocumentWithAst("x = 1")); - var symbols = await t; - symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)), - }); + index.Add(path, DocumentWithAst("x = 1")); + index.MarkAsPending(path); + var t = index.WorkspaceSymbolsAsync("", maxSymbols); + index.ReIndex(path, DocumentWithAst("x = 1")); + var symbols = await t; + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, path, new SourceSpan(1, 1, 1, 2)) + }); + } } private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) @@ -199,14 +207,14 @@ private IReadOnlyList WalkSymbols(string code, PythonLanguag private IDocument DocumentWithAst(string testCode, string filePath = null) { filePath = filePath ?? $"{_rootPath}/{testCode}.py"; - IDocument doc = Substitute.For(); + var doc = Substitute.For(); doc.GetAstAsync().ReturnsForAnyArgs(Task.FromResult(MakeAst(testCode))); doc.Uri.Returns(new Uri(filePath)); return doc; } private PythonAst MakeAst(string testCode) { - PythonLanguageVersion latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); + var latestVersion = PythonVersions.LatestAvailable3X.Version.ToLanguageVersion(); return Parser.CreateParser(MakeStream(testCode), latestVersion).ParseFile(); } diff --git a/src/PLS.sln b/src/PLS.sln index ab969a558..2bceb6b86 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -27,6 +27,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.T EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Tests", "LanguageServer\Test\Microsoft.Python.LanguageServer.Tests.csproj", "{3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Caching", "Caching\Impl\Microsoft.Python.Analysis.Caching.csproj", "{42BD3C80-3E57-4847-8142-84F6B682EA8D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Caching.Tests", "Caching\Test\Microsoft.Python.Analysis.Caching.Tests.csproj", "{40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -73,6 +77,14 @@ Global {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Debug|Any CPU.Build.0 = Debug|Any CPU {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.ActiveCfg = Release|Any CPU {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.Build.0 = Release|Any CPU + {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Release|Any CPU.Build.0 = Release|Any CPU + {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -88,6 +100,8 @@ Global {2C8DE250-41F4-4FC5-A661-76E2A4172891} = {C465393D-145E-4695-A7DB-AF55951BD533} {D8D85896-5DB0-4FA6-B744-910A272C39F9} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} + {42BD3C80-3E57-4847-8142-84F6B682EA8D} = {C465393D-145E-4695-A7DB-AF55951BD533} + {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ABC12ED7-0EC8-4219-8A14-A058F7942D92} diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 70a669cf9..5ebeee6e7 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -25,7 +25,7 @@ namespace Microsoft.Python.Parsing.Ast { /// /// Top-level ast for all Python code. Holds onto the body and the line mapping information. /// - public sealed class PythonAst : ScopeStatement { + public sealed class PythonAst : ScopeStatement, ILocationConverter { private readonly object _lock = new object(); private readonly Statement _body; private readonly Dictionary> _attributes = new Dictionary>(); @@ -140,8 +140,10 @@ internal void SetAttributes(Dictionary> attribu } } + #region ILocationConverter public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); + #endregion internal int GetLineEndFromPosition(int index) { var loc = IndexToLocation(index); diff --git a/src/Parsing/Impl/Ast/SourceLocationExtensions.cs b/src/Parsing/Impl/Ast/SourceLocationExtensions.cs index 4c8df3e07..9377fe70f 100644 --- a/src/Parsing/Impl/Ast/SourceLocationExtensions.cs +++ b/src/Parsing/Impl/Ast/SourceLocationExtensions.cs @@ -18,18 +18,18 @@ namespace Microsoft.Python.Parsing.Ast { public static class SourceLocationExtensions { - public static int ToIndex(this SourceLocation location, PythonAst ast) => ast.LocationToIndex(location); + public static int ToIndex(this SourceLocation location, ILocationConverter lc) => lc.LocationToIndex(location); } public static class SourceSpanExtensions { - public static IndexSpan ToIndexSpan(this SourceSpan span, PythonAst ast) - => IndexSpan.FromBounds(ast.LocationToIndex(span.Start), ast.LocationToIndex(span.End)); - public static IndexSpan ToIndexSpan(this Range range, PythonAst ast) - => IndexSpan.FromBounds(ast.LocationToIndex(range.start), ast.LocationToIndex(range.end)); + public static IndexSpan ToIndexSpan(this SourceSpan span, ILocationConverter lc) + => IndexSpan.FromBounds(lc.LocationToIndex(span.Start), lc.LocationToIndex(span.End)); + public static IndexSpan ToIndexSpan(this Range range, ILocationConverter lc) + => IndexSpan.FromBounds(lc.LocationToIndex(range.start), lc.LocationToIndex(range.end)); } public static class IndexSpanExtensions { - public static SourceSpan ToSourceSpan(this IndexSpan span, PythonAst ast) - => ast != null ? new SourceSpan(ast.IndexToLocation(span.Start), ast.IndexToLocation(span.End)) : default; + public static SourceSpan ToSourceSpan(this IndexSpan span, ILocationConverter lc) + => lc != null ? new SourceSpan(lc.IndexToLocation(span.Start), lc.IndexToLocation(span.End)) : default; } } diff --git a/src/Parsing/Impl/Resources.Designer.cs b/src/Parsing/Impl/Resources.Designer.cs index 74403388e..507ec0381 100644 --- a/src/Parsing/Impl/Resources.Designer.cs +++ b/src/Parsing/Impl/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Microsoft.Python.Parsing { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { diff --git a/src/Parsing/Impl/Tokens/NewLineKind.cs b/src/Parsing/Impl/Tokens/NewLineKind.cs new file mode 100644 index 000000000..fb103bbd3 --- /dev/null +++ b/src/Parsing/Impl/Tokens/NewLineKind.cs @@ -0,0 +1,24 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Parsing { + public enum NewLineKind { + None, + LineFeed, + CarriageReturn, + CarriageReturnLineFeed + } +} diff --git a/src/Parsing/Impl/Tokens/NewLineKindExtensions.cs b/src/Parsing/Impl/Tokens/NewLineKindExtensions.cs new file mode 100644 index 000000000..1c7e26803 --- /dev/null +++ b/src/Parsing/Impl/Tokens/NewLineKindExtensions.cs @@ -0,0 +1,39 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.Python.Parsing { + public static class NewLineKindExtensions { + public static int GetSize(this NewLineKind kind) { + switch (kind) { + case NewLineKind.LineFeed: return 1; + case NewLineKind.CarriageReturnLineFeed: return 2; + case NewLineKind.CarriageReturn: return 1; + } + return 0; + } + + public static string GetString(this NewLineKind kind) { + switch (kind) { + case NewLineKind.CarriageReturn: return "\r"; + case NewLineKind.CarriageReturnLineFeed: return "\r\n"; + case NewLineKind.LineFeed: return "\n"; + } + throw new InvalidOperationException(); + } + } +} diff --git a/src/Parsing/Impl/Tokens/NewLineLocation.cs b/src/Parsing/Impl/Tokens/NewLineLocation.cs new file mode 100644 index 000000000..478f83c89 --- /dev/null +++ b/src/Parsing/Impl/Tokens/NewLineLocation.cs @@ -0,0 +1,120 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.Parsing { + [DebuggerDisplay("NewLineLocation({EndIndex}, {Kind})")] + public struct NewLineLocation : IComparable { + public NewLineLocation(int lineEnd, NewLineKind kind) { + EndIndex = lineEnd; + Kind = kind; + } + + /// + /// The end of of the line, including the line break. + /// + public int EndIndex { get; } + + /// + /// The type of new line which terminated the line. + /// + public NewLineKind Kind { get; } + + public int CompareTo(NewLineLocation other) => EndIndex - other.EndIndex; + + public static SourceLocation IndexToLocation(NewLineLocation[] lineLocations, int index) { + if (lineLocations == null || index == 0) { + return new SourceLocation(index, 1, 1); + } + + var match = Array.BinarySearch(lineLocations, new NewLineLocation(index, NewLineKind.None)); + if (match < 0) { + // If our index = -1, it means we're on the first line. + if (match == -1) { + return new SourceLocation(index, 1, checked(index + 1)); + } + // If we couldn't find an exact match for this line number, get the nearest + // matching line number less than this one + match = ~match - 1; + } + + while (match >= 0 && index == lineLocations[match].EndIndex && lineLocations[match].Kind == NewLineKind.None) { + match -= 1; + } + if (match < 0) { + return new SourceLocation(index, 1, checked(index + 1)); + } + + var line = match + 2; + var col = index - lineLocations[match].EndIndex + 1; + return new SourceLocation(index, line, col); + } + + public static int LocationToIndex(NewLineLocation[] lineLocations, SourceLocation location, int endIndex) { + if (lineLocations == null) { + return 0; + } + var index = 0; + if (lineLocations.Length == 0) { + // We have a single line, so the column is the index + index = location.Column - 1; + return endIndex >= 0 ? Math.Min(index, endIndex) : index; + } + var line = location.Line - 1; + + if (line > lineLocations.Length) { + index = lineLocations[lineLocations.Length - 1].EndIndex; + return endIndex >= 0 ? Math.Min(index, endIndex) : index; + } + + if (line > 0) { + index = lineLocations[line - 1].EndIndex; + } + + if (line < lineLocations.Length && location.Column > (lineLocations[line].EndIndex - index)) { + index = lineLocations[line].EndIndex; + return endIndex >= 0 ? Math.Min(index, endIndex) : index; + } + + if (endIndex < 0) { + endIndex = lineLocations[lineLocations.Length - 1].EndIndex; + } + + return (int)Math.Min((long)index + location.Column - 1, endIndex); + } + + private static readonly char[] _lineSeparators = new[] { '\r', '\n' }; + + public static NewLineLocation FindNewLine(string text, int start) { + var i = text.IndexOfAny(_lineSeparators, start); + if (i < start) { + return new NewLineLocation(text.Length, NewLineKind.None); + } + if (text[i] == '\n') { + return new NewLineLocation(i + 1, NewLineKind.LineFeed); + } + if (text.Length > i + 1 && text[i + 1] == '\n') { + return new NewLineLocation(i + 2, NewLineKind.CarriageReturnLineFeed); + } + return new NewLineLocation(i + 1, NewLineKind.CarriageReturn); + } + + public override string ToString() => $""; + } +} diff --git a/src/Parsing/Impl/Token.cs b/src/Parsing/Impl/Tokens/Token.cs similarity index 100% rename from src/Parsing/Impl/Token.cs rename to src/Parsing/Impl/Tokens/Token.cs diff --git a/src/Parsing/Impl/TokenCategory.cs b/src/Parsing/Impl/Tokens/TokenCategory.cs similarity index 100% rename from src/Parsing/Impl/TokenCategory.cs rename to src/Parsing/Impl/Tokens/TokenCategory.cs diff --git a/src/Parsing/Impl/TokenInfo.cs b/src/Parsing/Impl/Tokens/TokenInfo.cs similarity index 100% rename from src/Parsing/Impl/TokenInfo.cs rename to src/Parsing/Impl/Tokens/TokenInfo.cs diff --git a/src/Parsing/Impl/TokenKind.Generated.cs b/src/Parsing/Impl/Tokens/TokenKind.Generated.cs similarity index 100% rename from src/Parsing/Impl/TokenKind.Generated.cs rename to src/Parsing/Impl/Tokens/TokenKind.Generated.cs diff --git a/src/Parsing/Impl/TokenTriggers.cs b/src/Parsing/Impl/Tokens/TokenTriggers.cs similarity index 100% rename from src/Parsing/Impl/TokenTriggers.cs rename to src/Parsing/Impl/Tokens/TokenTriggers.cs diff --git a/src/Parsing/Impl/Tokenizer.cs b/src/Parsing/Impl/Tokens/Tokenizer.cs similarity index 95% rename from src/Parsing/Impl/Tokenizer.cs rename to src/Parsing/Impl/Tokens/Tokenizer.cs index e77356a9a..ec5622252 100644 --- a/src/Parsing/Impl/Tokenizer.cs +++ b/src/Parsing/Impl/Tokens/Tokenizer.cs @@ -28,7 +28,6 @@ using Microsoft.Python.Core.Text; namespace Microsoft.Python.Parsing { - /// /// IronPython tokenizer /// @@ -2585,133 +2584,4 @@ private void ClearInvalidChars() { #endregion } - - public enum NewLineKind { - None, - LineFeed, - CarriageReturn, - CarriageReturnLineFeed - } - - [DebuggerDisplay("NewLineLocation({_endIndex}, {_kind})")] - public struct NewLineLocation : IComparable { - private readonly int _endIndex; - private readonly NewLineKind _kind; - - public NewLineLocation(int lineEnd, NewLineKind kind) { - _endIndex = lineEnd; - _kind = kind; - } - - /// - /// The end of of the line, including the line break. - /// - public int EndIndex => _endIndex; - - /// - /// The type of new line which terminated the line. - /// - public NewLineKind Kind => _kind; - - public int CompareTo(NewLineLocation other) => EndIndex - other.EndIndex; - - public static SourceLocation IndexToLocation(NewLineLocation[] lineLocations, int index) { - if (lineLocations == null || index == 0) { - return new SourceLocation(index, 1, 1); - } - - var match = Array.BinarySearch(lineLocations, new NewLineLocation(index, NewLineKind.None)); - if (match < 0) { - // If our index = -1, it means we're on the first line. - if (match == -1) { - return new SourceLocation(index, 1, checked(index + 1)); - } - // If we couldn't find an exact match for this line number, get the nearest - // matching line number less than this one - match = ~match - 1; - } - - while (match >= 0 && index == lineLocations[match].EndIndex && lineLocations[match].Kind == NewLineKind.None) { - match -= 1; - } - if (match < 0) { - return new SourceLocation(index, 1, checked(index + 1)); - } - - var line = match + 2; - var col = index - lineLocations[match].EndIndex + 1; - return new SourceLocation(index, line, col); - } - - public static int LocationToIndex(NewLineLocation[] lineLocations, SourceLocation location, int endIndex) { - if (lineLocations == null) { - return 0; - } - var index = 0; - if (lineLocations.Length == 0) { - // We have a single line, so the column is the index - index = location.Column - 1; - return endIndex >= 0 ? Math.Min(index, endIndex) : index; - } - var line = location.Line - 1; - - if (line > lineLocations.Length) { - index = lineLocations[lineLocations.Length - 1].EndIndex; - return endIndex >= 0 ? Math.Min(index, endIndex) : index; - } - - if (line > 0) { - index = lineLocations[line - 1].EndIndex; - } - - if (line < lineLocations.Length && location.Column > (lineLocations[line].EndIndex - index)) { - index = lineLocations[line].EndIndex; - return endIndex >= 0 ? Math.Min(index, endIndex) : index; - } - - if (endIndex < 0) { - endIndex = lineLocations[lineLocations.Length - 1].EndIndex; - } - - return (int)Math.Min((long)index + location.Column - 1, endIndex); - } - - private static readonly char[] _lineSeparators = new[] { '\r', '\n' }; - - public static NewLineLocation FindNewLine(string text, int start) { - var i = text.IndexOfAny(_lineSeparators, start); - if (i < start) { - return new NewLineLocation(text.Length, NewLineKind.None); - } - if (text[i] == '\n') { - return new NewLineLocation(i + 1, NewLineKind.LineFeed); - } - if (text.Length > i + 1 && text[i + 1] == '\n') { - return new NewLineLocation(i + 2, NewLineKind.CarriageReturnLineFeed); - } - return new NewLineLocation(i + 1, NewLineKind.CarriageReturn); - } - - public override string ToString() => $""; - } - - public static class NewLineKindExtensions { - public static int GetSize(this NewLineKind kind) { - switch (kind) { - case NewLineKind.LineFeed: return 1; - case NewLineKind.CarriageReturnLineFeed: return 2; - case NewLineKind.CarriageReturn: return 1; - } - return 0; - } - - public static string GetString(this NewLineKind kind) { - switch (kind) { - case NewLineKind.CarriageReturn: return "\r"; - case NewLineKind.CarriageReturnLineFeed: return "\r\n"; - case NewLineKind.LineFeed: return "\n"; - } - throw new InvalidOperationException(); - } - } } diff --git a/src/Parsing/Impl/TokenizerOptions.cs b/src/Parsing/Impl/Tokens/TokenizerOptions.cs similarity index 100% rename from src/Parsing/Impl/TokenizerOptions.cs rename to src/Parsing/Impl/Tokens/TokenizerOptions.cs diff --git a/src/Parsing/Test/PythonInstallPathResolver.cs b/src/Parsing/Test/PythonInstallPathResolver.cs index 31676648e..37fae5fb2 100644 --- a/src/Parsing/Test/PythonInstallPathResolver.cs +++ b/src/Parsing/Test/PythonInstallPathResolver.cs @@ -37,6 +37,5 @@ public static class PythonInstallPathResolver { public interface IPythonInstallPathResolver { InterpreterConfiguration GetCorePythonConfiguration(InterpreterArchitecture architecture, Version version); InterpreterConfiguration GetCondaPythonConfiguration(InterpreterArchitecture architecture, Version version); - InterpreterConfiguration GetIronPythonConfiguration(bool x64); } } diff --git a/src/Parsing/Test/PythonVersion.cs b/src/Parsing/Test/PythonVersion.cs index a0d7be0ac..612db44f4 100644 --- a/src/Parsing/Test/PythonVersion.cs +++ b/src/Parsing/Test/PythonVersion.cs @@ -25,11 +25,9 @@ public PythonVersion(InterpreterConfiguration config, bool cPython = false) { IsCPython = cPython; } - public override string ToString() => Configuration.Description; public string LibraryPath => Configuration.LibraryPath; public string InterpreterPath => Configuration.InterpreterPath; public PythonLanguageVersion Version => Configuration.Version.ToLanguageVersion(); - public string Id => Configuration.Id; public bool Isx64 => Configuration.Architecture == InterpreterArchitecture.x64; public InterpreterArchitecture Architecture => Configuration.Architecture; } diff --git a/src/Parsing/Test/PythonVersions.cs b/src/Parsing/Test/PythonVersions.cs index 771aef56b..a199eba79 100644 --- a/src/Parsing/Test/PythonVersions.cs +++ b/src/Parsing/Test/PythonVersions.cs @@ -26,7 +26,6 @@ public static class PythonVersions { public static readonly InterpreterConfiguration Python36 = GetCPythonVersion(PythonLanguageVersion.V36, InterpreterArchitecture.x86); public static readonly InterpreterConfiguration Python37 = GetCPythonVersion(PythonLanguageVersion.V37, InterpreterArchitecture.x86); public static readonly InterpreterConfiguration Python38 = GetCPythonVersion(PythonLanguageVersion.V38, InterpreterArchitecture.x86); - public static readonly InterpreterConfiguration IronPython27 = GetIronPythonVersion(false); public static readonly InterpreterConfiguration Python27_x64 = GetCPythonVersion(PythonLanguageVersion.V27, InterpreterArchitecture.x64); public static readonly InterpreterConfiguration Python35_x64 = GetCPythonVersion(PythonLanguageVersion.V35, InterpreterArchitecture.x64); public static readonly InterpreterConfiguration Python36_x64 = GetCPythonVersion(PythonLanguageVersion.V36, InterpreterArchitecture.x64); @@ -38,8 +37,6 @@ public static class PythonVersions { public static readonly InterpreterConfiguration Anaconda36_x64 = GetAnacondaVersion(PythonLanguageVersion.V36, InterpreterArchitecture.x64); public static readonly InterpreterConfiguration Anaconda37 = GetAnacondaVersion(PythonLanguageVersion.V37, InterpreterArchitecture.x86); public static readonly InterpreterConfiguration Anaconda37_x64 = GetAnacondaVersion(PythonLanguageVersion.V37, InterpreterArchitecture.x64); - public static readonly InterpreterConfiguration IronPython27_x64 = GetIronPythonVersion(true); - public static readonly InterpreterConfiguration Jython27 = GetJythonVersion(PythonLanguageVersion.V27); public static IEnumerable AnacondaVersions => GetVersions( Anaconda36, @@ -52,8 +49,6 @@ public static class PythonVersions { public static IEnumerable Versions => GetVersions( Python27, Python27_x64, - IronPython27, - IronPython27_x64, Python35, Python35_x64, Python36, @@ -61,7 +56,7 @@ public static class PythonVersions { Python37, Python37_x64, Python38, - Python38_x64, Jython27); + Python38_x64); public static InterpreterConfiguration Required_Python27X => Python27 ?? Python27_x64 ?? NotInstalled("v2.7"); public static InterpreterConfiguration Required_Python35X => Python35 ?? Python35_x64 ?? NotInstalled("v3.5"); @@ -122,50 +117,6 @@ private static InterpreterConfiguration GetCPythonVersion(PythonLanguageVersion private static InterpreterConfiguration GetAnacondaVersion(PythonLanguageVersion version, InterpreterArchitecture arch) => PythonInstallPathResolver.Instance.GetCondaPythonConfiguration(arch, version.ToVersion()); - private static InterpreterConfiguration GetIronPythonVersion(bool x64) { - return PythonInstallPathResolver.Instance.GetIronPythonConfiguration(x64); - } - - private static InterpreterConfiguration GetJythonVersion(PythonLanguageVersion version) { - var candidates = new List(); - var ver = version.ToVersion(); - var path1 = string.Format("jython{0}{1}*", ver.Major, ver.Minor); - var path2 = string.Format("jython{0}.{1}*", ver.Major, ver.Minor); - - foreach (var drive in DriveInfo.GetDrives()) { - if (drive.DriveType != DriveType.Fixed) { - continue; - } - - try { - candidates.AddRange(drive.RootDirectory.EnumerateDirectories(path1)); - candidates.AddRange(drive.RootDirectory.EnumerateDirectories(path2)); - } catch { - } - } - - foreach (var dir in candidates) { - var interpreter = dir.EnumerateFiles("jython.bat").FirstOrDefault(); - if (interpreter == null) { - continue; - } - - var libPath = dir.EnumerateDirectories("Lib").FirstOrDefault(); - if (libPath == null || !libPath.EnumerateFiles("site.py").Any()) { - continue; - } - - return new InterpreterConfiguration( - id: $"Global|Jython|{version.ToVersion()}", - description: string.Format("Jython {0}", version.ToVersion()), - interpreterPath: interpreter.FullName, - version: version.ToVersion() - ); - } - - return null; - } - private static InterpreterConfiguration NotInstalled(string version) { Assert.Inconclusive($"Python interpreter {version} is not installed"); return null; diff --git a/src/Parsing/Test/UnixPythonInstallPathResolver.cs b/src/Parsing/Test/UnixPythonInstallPathResolver.cs index 1cdf77605..4931a52b7 100644 --- a/src/Parsing/Test/UnixPythonInstallPathResolver.cs +++ b/src/Parsing/Test/UnixPythonInstallPathResolver.cs @@ -40,8 +40,6 @@ public InterpreterConfiguration GetCorePythonConfiguration(InterpreterArchitectu public InterpreterConfiguration GetCondaPythonConfiguration(InterpreterArchitecture architecture, Version version) => architecture == InterpreterArchitecture.x86 ? null : _condaCache.TryGetValue(version, out var interpreterConfiguration) ? interpreterConfiguration : null; - public InterpreterConfiguration GetIronPythonConfiguration(bool x64) => null; - private void GetConfigurationsFromKnownPaths() { var homePath = Environment.GetEnvironmentVariable("HOME"); var foldersFromPathVariable = Environment.GetEnvironmentVariable("PATH")?.Split(':') ?? Array.Empty(); @@ -90,8 +88,6 @@ private InterpreterConfiguration GetConfiguration(string idPrefix, string python var sitePackagesPath = GetSitePackagesLocation(pythonFilePath); return new InterpreterConfiguration( - id: $"{idPrefix}|{version}", - description: $"{idPrefix} {version} ({architecture})", interpreterPath: pythonFilePath, pathVar: pythonFilePath, libPath: libPath, diff --git a/src/Parsing/Test/WindowsPythonInstallPathResolver.cs b/src/Parsing/Test/WindowsPythonInstallPathResolver.cs index 9ec3a57fe..2a77e4ded 100644 --- a/src/Parsing/Test/WindowsPythonInstallPathResolver.cs +++ b/src/Parsing/Test/WindowsPythonInstallPathResolver.cs @@ -40,34 +40,9 @@ public InterpreterConfiguration GetCondaPythonConfiguration(InterpreterArchitect private InterpreterConfiguration GetPythonConfiguration(string prefix, InterpreterArchitecture architecture, Version version) => _registryCache.FirstOrDefault(configuration => - configuration.Id.StartsWith(prefix) && configuration.Architecture == architecture && configuration.Version == version); - public InterpreterConfiguration GetIronPythonConfiguration(bool x64) { - var installPath = GetIronPythonInstallDir(); - if (!Directory.Exists(installPath)) { - return null; - } - - var exeName = x64 ? "ipy64.exe" : "ipy.exe"; - // IronPython changed to Any CPU for ipy.exe and ipy32.exe for 32-bit in 2.7.8 - if (File.Exists(Path.Combine(installPath, "ipy32.exe"))) { - exeName = x64 ? "ipy.exe" : "ipy32.exe"; - } - - return new InterpreterConfiguration( - id: x64 ? "IronPython|2.7-64" : "IronPython|2.7-32", - description: string.Format("IronPython {0} 2.7", x64 ? "64-bit" : "32-bit"), - interpreterPath: Path.Combine(installPath, exeName), - libPath: Path.Combine(installPath, "Lib"), - sitePackagesPath: Path.Combine(installPath, "Lib", "site-packages"), - architecture: x64 ? InterpreterArchitecture.x64 : InterpreterArchitecture.x86, - version: new Version(2, 7), - pathVar: "IRONPYTHONPATH" - ); - } - private List FindPythonConfigurationsInRegistry() { var configurations = new List(); @@ -87,7 +62,6 @@ private List FindPythonConfigurationsInRegistry() { } } - InterpreterConfiguration.DisambiguateDescriptions(configurations); return configurations; } @@ -183,16 +157,8 @@ InterpreterArchitecture assumedArchitecture } var pathVar = tagKey.GetValue("PathEnvironmentVariable") as string ?? "PYTHONPATH"; - var id = $"Global|{company}|{tag}"; - - var description = tagKey.GetValue("DisplayName") as string; - if (string.IsNullOrEmpty(description)) { - description = pythonCoreCompatibility ? "Python {0}{1: ()}".FormatUI(version, architecture) : "{0} {1}".FormatUI(company, tag); - } return new InterpreterConfiguration( - id: id, - description: description, interpreterPath: exePath, pathVar: pathVar, libPath: Path.Combine(prefixPath, "Lib"), diff --git a/src/UnitTests/Core/Impl/Baseline.cs b/src/UnitTests/Core/Impl/Baseline.cs new file mode 100644 index 000000000..74e87d235 --- /dev/null +++ b/src/UnitTests/Core/Impl/Baseline.cs @@ -0,0 +1,125 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text; +using FluentAssertions; +using FluentAssertions.Execution; + +namespace TestUtilities { + [ExcludeFromCodeCoverage] + public static class Baseline { + public static string CompareStrings(string expected, string actual) { + var result = new StringBuilder(); + + var length = Math.Min(expected.Length, actual.Length); + for (var i = 0; i < length; i++) { + if (expected[i] != actual[i]) { + result.AppendLine(FormattableString.Invariant($"Position: {i}: expected: '{expected[i]}', actual '{actual[i]}'")); + if (i > 6 && i < length - 6) { + result.Append(FormattableString.Invariant($"Context: {expected.Substring(i - 6, 12)} -> {actual.Substring(i - 6, 12)}")); + } + break; + } + + } + + if (expected.Length != actual.Length) { + result.Append(FormattableString.Invariant($"\r\nLength different. Expected: '{expected.Length}' , actual '{actual.Length}'")); + } + + return result.ToString(); + } + + public static void CompareStringLines(string expected, string actual) { + var line = CompareLines(expected, actual, out var baseLine, out var actualLine, out var index); + line.Should().Be(0, $@"there should be no difference at line {line} + Expected:{baseLine.Trim()} + Actual:{actualLine.Trim()} + Difference at position {index}{Environment.NewLine}"); + } + + public static int CompareLines(string expected, string actual, out string expectedLine, out string actualLine, out int index, bool ignoreCase = false) { + using (var actualReader = new StringReader(actual)) + using (var expectedReader = new StringReader(expected)) { + + var lineNum = 1; + index = 0; + + for (;; lineNum++) { + expectedLine = expectedReader.ReadLine(); + actualLine = actualReader.ReadLine(); + + if (expectedLine == null || actualLine == null) { + break; + } + + var minLength = Math.Min(expectedLine.Length, actualLine.Length); + for (var i = 0; i < minLength; i++) { + var act = actualLine[i]; + var exp = expectedLine[i]; + + if (ignoreCase) { + act = char.ToLowerInvariant(act); + exp = char.ToLowerInvariant(exp); + } + + if (act != exp) { + index = i + 1; + return lineNum; + } + } + + if (expectedLine.Length != actualLine.Length) { + index = minLength + 1; + return lineNum; + } + } + + if (expectedLine == null && actualLine == null) { + expectedLine = string.Empty; + actualLine = string.Empty; + + return 0; + } + + return lineNum; + } + } + + public static void CompareToFile(string baselineFile, string actual, bool regenerateBaseline = false, bool ignoreCase = false) { + if (regenerateBaseline) { + if (File.Exists(baselineFile)) { + File.SetAttributes(baselineFile, FileAttributes.Normal); + } + + File.WriteAllText(baselineFile, actual.Trim()); + return; + } + + actual = actual.Trim(); + var expected = File.ReadAllText(baselineFile).Trim(); + var line = CompareLines(expected, actual, out var baseLine, out var actualLine, out var index, ignoreCase); + Execute.Assertion.ForCondition(line == 0) + .FailWith($@"there should be no difference at line {line} + Expected:{baseLine.Trim()} + Actual:{actualLine.Trim()} + BaselineFile:{Path.GetFileName(baselineFile)} + Difference at column {index}{Environment.NewLine}"); + } + } +}