diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 3de3ac411..a2d508cd0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -40,6 +40,7 @@ private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool is Name = name; FilePath = filePath; IsTypeshed = isTypeshed; + IsNonUserAsDocument = isNonUserAsDocument; } public AnalysisModuleKey GetNonUserAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, IsTypeshed, true); @@ -70,6 +71,8 @@ public void Deconstruct(out string moduleName, out string filePath, out bool isT public override string ToString() => $"{Name}({FilePath})"; + public bool IsNonUserAsDocument { get; } + private static bool IsNonUserAsDocumentModule(IPythonModule module) => (module.IsNonUserFile() || module.IsCompiled()) && module is IDocument document && document.IsOpen; } diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index f85643fba..ed80d9018 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -40,9 +40,9 @@ internal abstract class AnalysisWalker : PythonWalker { public PythonAst Ast => Eval.Ast; protected ModuleSymbolTable SymbolTable => Eval.SymbolTable; - protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) { + protected AnalysisWalker(ExpressionEval eval) { Eval = eval; - ImportHandler = new ImportHandler(this, importedVariableHandler); + ImportHandler = new ImportHandler(this); AssignmentHandler = new AssignmentHandler(this); LoopHandler = new LoopHandler(this); ConditionalHandler = new ConditionalHandler(this); diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 9a3b9007d..b13891861 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -21,6 +21,11 @@ namespace Microsoft.Python.Analysis.Analyzer { /// Represents document that can be analyzed asynchronously. /// internal interface IAnalyzable { + /// + /// Returns object that can calculate dependencies of this entry. + /// + IDependencyProvider DependencyProvider { get; } + /// /// Notifies document that analysis is about to begin. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index f55abb0f6..03a37a1c2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -60,8 +60,8 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { - if (node == null || - Module.ModuleType == ModuleType.Specialized || Module.ModuleType == ModuleType.Compiled || + if (node == null || + Module.ModuleType == ModuleType.Specialized || Module.ModuleType == ModuleType.Compiled || Module.ModuleType == ModuleType.CompiledBuiltin || Module.ModuleType == ModuleType.Builtins) { return DefaultLocation; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs index ca1931259..2790dee88 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; @@ -29,7 +28,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker { private readonly FunctionDefinition _function; private IMember _result; - public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) { + public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) { _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); _eval = eval ?? throw new ArgumentNullException(nameof(eval)); _function = fd ?? throw new ArgumentNullException(nameof(fd)); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index f57244f9e..3ea2f5411 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -68,13 +68,6 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor if (!string.IsNullOrEmpty(variableName)) { DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression); } - - if (imports is IImportChildrenSource cs - && cs.TryGetChildImport(memberName, out var csr) - && HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) { - - _importedVariableHandler.EnsureModule(childModule); - } } } @@ -83,11 +76,11 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport // from self import * won't define any new members return; } - // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. var memberNames = imports is ImplicitPackageImport ? variableModule.GetMemberNames() - : _importedVariableHandler.GetMemberNames(variableModule).ToArray(); + : variableModule.Analysis.StarImportMemberNames + ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")).ToArray(); foreach (var memberName in memberNames) { DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression); @@ -114,27 +107,14 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); // First try exported or child submodules. - var member = variableModule.GetMember(memberName); + value = value ?? variableModule.GetMember(memberName); // Value may be variable or submodule. If it is variable, we need it in order to add reference. - var variable = _importedVariableHandler.GetVariable(variableModule, memberName); - - if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) { - // If member is submodule, use actual variable so it can be linked through for goto definition. - value = variable; - } else if (value == null) { - if (member is PythonVariableModule) { - // If member is submodule, use it. - value = member; - } else if (variable?.Value != null) { - // Otherwise use variable, if available so references can be linked. - value = variable; - } else if (member != null) { - value = member; - } else { - value = Eval.UnknownType; - } - } + var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; + value = variable?.Value?.Equals(value) == true ? variable : value; + + // If nothing is exported, variables are still accessible. + value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; } // Do not allow imported variables to override local declarations diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 66e0cdc0b..4cdf23d8b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -29,12 +29,9 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers { internal sealed partial class ImportHandler : StatementHandler { - private readonly IImportedVariableHandler _importedVariableHandler; private readonly Dictionary _variableModules = new Dictionary(); - public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) { - _importedVariableHandler = importedVariableHandler; - } + public ImportHandler(AnalysisWalker walker) : base(walker) { } public bool HandleImport(ImportStatement node) { if (Module.ModuleType == ModuleType.Specialized) { @@ -68,9 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa lastModule = default; break; } - resolvedModules[i] = (nameExpression.Name, lastModule); - _importedVariableHandler.EnsureModule(lastModule); } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') @@ -94,6 +89,15 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); } } + + // import a.b.c.d => declares a, b in the current module, c in b, d in c. + for (var i = 1; i < resolvedModules.Length - 1; i++) { + var (childName, childModule) = resolvedModules[i + 1]; + if (!string.IsNullOrEmpty(childName) && childModule != null) { + var parent = resolvedModules[i].module; + parent?.AddChildModule(childName, childModule); + } + } } private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) { @@ -108,7 +112,8 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV return TryGetPackageFromImport(packageImport, parent, out variableModule); case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); - Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + Eval.ReportDiagnostics(Eval.Module.Uri, + new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; case ImportNotFound importNotFound: @@ -172,7 +177,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor return false; } } - + return true; } @@ -195,7 +200,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py variableModule = new PythonVariableModule(fullName, Eval.Interpreter); _variableModules[fullName] = variableModule; } - + parentModule?.AddChildModule(memberName, variableModule); return variableModule; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs deleted file mode 100644 index ea93c71cf..000000000 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal sealed class LoopImportedVariableHandler : IImportedVariableHandler { - private readonly Dictionary _walkers = new Dictionary(); - private readonly IReadOnlyDictionary _asts; - private readonly IReadOnlyDictionary _cachedVariables; - private readonly IServiceContainer _services; - private readonly Func _isCanceled; - - public IReadOnlyCollection Walkers => _walkers.Values; - - public LoopImportedVariableHandler(in IServiceContainer services, - in IReadOnlyDictionary asts, - in IReadOnlyDictionary cachedVariables, - in Func isCanceled) { - - _services = services; - _isCanceled = isCanceled; - _asts = asts; - _cachedVariables = cachedVariables; - } - - public IEnumerable GetMemberNames(PythonVariableModule variableModule) { - var module = variableModule.Module; - if (module == null || _isCanceled()) { - return Enumerable.Empty(); - } - - var key = new AnalysisModuleKey(module); - if (_walkers.TryGetValue(key, out var walker)) { - return GetMemberNames(walker, variableModule); - } - - if (!_asts.TryGetValue(key, out var ast)) { - return _cachedVariables.TryGetValue(key, out var variables) - ? variables.Names - : variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); - } - - walker = WalkModule(module, ast); - return walker != null ? GetMemberNames(walker, variableModule) : module.GetMemberNames(); - } - - public IVariable GetVariable(in PythonVariableModule variableModule, in string name) { - var module = variableModule.Module; - if (module == null || _isCanceled()) { - return default; - } - - var key = new AnalysisModuleKey(module); - if (_walkers.TryGetValue(key, out var walker)) { - return walker.Eval.GlobalScope?.Variables[name]; - } - - if (!_asts.TryGetValue(key, out var ast)) { - return _cachedVariables.TryGetValue(key, out var variables) ? variables[name] : default; - } - - _walkers[key] = walker = WalkModule(module, ast); - var gs = walker != null ? walker.Eval.GlobalScope : module.GlobalScope; - return gs?.Variables[name]; - } - - public void EnsureModule(in PythonVariableModule variableModule) { - var module = variableModule.Module; - if (module == null || _isCanceled()) { - return; - } - EnsureModule(module); - } - - private void EnsureModule(in IPythonModule module) { - if (module == null || _isCanceled()) { - return; - } - var key = new AnalysisModuleKey(module); - if (!_walkers.ContainsKey(key) && _asts.TryGetValue(key, out var ast)) { - WalkModule(module, ast); - } - } - - public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) { - // If module has stub, make sure it is processed too. - if (module.Stub?.Analysis is EmptyAnalysis && module.Stub.GetAst() != null) { - WalkModule(module.Stub, module.Stub.GetAst()); - } - - var eval = new ExpressionEval(_services, module, ast); - var walker = new ModuleWalker(eval, this); - - _walkers[new AnalysisModuleKey(module)] = walker; - ast.Walk(walker); - walker.Complete(); - return walker; - } - - private static IEnumerable GetMemberNames(ModuleWalker walker, PythonVariableModule variableModule) - => walker.StarImportMemberNames ?? walker.GlobalScope.GetExportableVariableNames().Concat(variableModule.ChildrenNames).Distinct().Where(s => !s.StartsWithOrdinal("_")); - } -} diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs deleted file mode 100644 index 86d08832e..000000000 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; - -namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal sealed class SimpleImportedVariableHandler : IImportedVariableHandler { - public static IImportedVariableHandler Instance { get; } = new SimpleImportedVariableHandler(); - - private SimpleImportedVariableHandler() {} - - public IEnumerable GetMemberNames(PythonVariableModule variableModule) - => variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); - - public IVariable GetVariable(in PythonVariableModule module, in string memberName) - => module.Analysis?.GlobalScope?.Variables[memberName]; - - public void EnsureModule(in PythonVariableModule module) { } - } -} diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 201f7660c..68f7a1cb1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -18,7 +18,6 @@ using System.Linq; using System.Threading; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; @@ -38,7 +37,8 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) : base(eval, importedVariableHandler) { + public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken) + : base(new ExpressionEval(services, module, ast)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; _cancellationToken = CancellationToken.None; } @@ -206,7 +206,7 @@ public void Complete() { new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken); if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable) - && variable.Value is IPythonCollection collection && collection.IsExact) { + && variable?.Value is IPythonCollection collection && collection.IsExact) { StarImportMemberNames = collection.Contents .OfType() .Select(c => c.GetString()) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 9822c74c8..c784d77b8 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -155,7 +155,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int } } - var invalidate = entry.Invalidate(module, ast, bufferVersion, version, out var dependencies) || module.Analysis is EmptyAnalysis; + var invalidate = entry.Invalidate(module, ast, bufferVersion, version, out var dependencies); if (invalidate) { AnalyzeDocument(key, entry, dependencies); } @@ -186,13 +186,13 @@ public async Task ResetAnalyzer() { lock (_syncObj) { _forceGCOnNextSession = true; - _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out _); + _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); _analysisEntries.Clear(); foreach (var (key, entry) in entriesToPreserve) { _analysisEntries.Add(key, entry); } - _dependencyResolver.Reset(); + _dependencyResolver.RemoveKeys(entriesToRemove.Select(e => e.Key)); } _services.GetService().ReloadAll(); @@ -212,30 +212,31 @@ public IReadOnlyList LoadedModules { internal async Task RaiseAnalysisCompleteAsync(int moduleCount, double msElapsed) { var notAllAnalyzed = false; lock (_syncObj) { - if (_nextSession != null || (_currentSession != null && !_currentSession.IsCompleted)) { - return false; // There are active or pending sessions. - } - if (_analysisEntries.Values.ExcludeDefault().Any(e => e.Module.ModuleState < ModuleState.Analyzing)) { - return false; // There are modules that are still being parsed. + if (_nextSession != null || _currentSession?.IsCompleted == false) { + return false; // There are active or pending sessions. } - var notAnalyzed = _analysisEntries.Values.ExcludeDefault().Where(e => e.NotAnalyzed).ToArray(); + var notAnalyzed = _analysisEntries.Values + .ExcludeDefault() + .Where(e => e.NotAnalyzed) + .ToArray(); + notAllAnalyzed = notAnalyzed.Length > 0; } if (notAllAnalyzed) { - // Attempt to see if within reasonable time new session starts - // This is a workaround since there may still be concurrency issues - // When module analysis session gets canceled and module never re-queued. - // We don't want to prevent event from firing when this [rarely] happens. + // Attempt to see if within reasonable time new session starts + // This is a workaround since there may still be concurrency issues + // When module analysis session gets canceled and module never re-queued. + // We don't want to prevent event from firing when this [rarely] happens. for (var i = 0; i < 20; i++) { await Task.Delay(20); lock (_syncObj) { - if(_analysisEntries.Values.ExcludeDefault().All(e => !e.NotAnalyzed)) { - break; // Now all modules are analyzed. + if (_analysisEntries.Values.ExcludeDefault().All(e => !e.NotAnalyzed)) { + break; // Now all modules are analyzed. } - if (_currentSession != null || _nextSession != null) { - return false; // New sessions were created + if (_nextSession != null || _currentSession?.IsCompleted == false) { + return false; // New sessions were created } } } @@ -243,7 +244,6 @@ internal async Task RaiseAnalysisCompleteAsync(int moduleCount, double msE _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); - return true; } @@ -253,10 +253,12 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}"); var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin, dependencies); + lock (_syncObj) { if (_version >= graphVersion) { return; } + _version = graphVersion; _currentSession?.Cancel(); } @@ -375,14 +377,10 @@ private PythonAnalyzerEntry GetOrCreateAnalysisEntry(IPythonModule module, Analy return entry; } - if (module.ModuleType == ModuleType.Specialized) { - entry = new PythonAnalyzerEntry(new CachedAnalysis((IDocument)module, _services)); - } else { - entry = new PythonAnalyzerEntry(new EmptyAnalysis(_services, (IDocument)module)); - _analysisCompleteEvent.Reset(); - } - + var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module); + entry = new PythonAnalyzerEntry(emptyAnalysis); _analysisEntries[key] = entry; + _analysisCompleteEvent.Reset(); return entry; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 90ddb0cb9..16ffd9b90 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -19,7 +19,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -82,27 +81,18 @@ public int AnalysisVersion { } } - public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis; + public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis && Module.ModuleType != ModuleType.Specialized && Module.ModuleType != ModuleType.Builtins; - public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) : this(emptyAnalysis.Document, emptyAnalysis) { - _bufferVersion = -1; - } - - public PythonAnalyzerEntry(CachedAnalysis analysis) : this(analysis.Document, analysis) { - _bufferVersion = 0; - _analysisTcs.SetResult(analysis); - } - - private PythonAnalyzerEntry(IPythonModule module, IDocumentAnalysis analysis) { - _previousAnalysis = analysis; - _module = module; + public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) { + _previousAnalysis = emptyAnalysis; + _module = emptyAnalysis.Document; _moduleType = _module.ModuleType; + _bufferVersion = -1; _analysisVersion = 0; _analysisTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } - public Task GetAnalysisAsync(CancellationToken cancellationToken) => _analysisTcs.Task.WaitAsync(cancellationToken); @@ -197,9 +187,8 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } var dependenciesHashSet = new HashSet(); - foreach (var dependency in analysisDependencies.ExcludeDefault().Where(d => d.ModuleType != ModuleType.Specialized)) { - if (!dependency.Equals(module) && - (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) { + foreach (var dependency in analysisDependencies) { + if (dependency != module && (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) { dependenciesHashSet.Add(new AnalysisModuleKey(dependency)); } } @@ -259,12 +248,15 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { var dependencies = new HashSet(); - if (_bufferVersion > bufferVersion || module.ModuleType == ModuleType.Specialized) { + if (_bufferVersion > bufferVersion) { return dependencies; } - var dw = new DependencyWalker(_module, ast); - dependencies.UnionWith(dw.Dependencies); + var dependencyProvider = (module as IAnalyzable)?.DependencyProvider; + var moduleDeps = dependencyProvider?.GetDependencies(ast); + if (moduleDeps != null) { + dependencies.UnionWith(moduleDeps); + } dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index f532f2037..989a66c7b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime; @@ -22,18 +21,14 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Caching; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.OS; using Microsoft.Python.Core.Testing; using Microsoft.Python.Parsing.Ast; @@ -48,17 +43,15 @@ internal sealed class PythonAnalyzerSession { private readonly CancellationToken _analyzerCancellationToken; private readonly IServiceContainer _services; private readonly IDiagnosticsService _diagnosticsService; - private readonly IOSPlatform _platformService; private readonly IProgressReporter _progress; private readonly PythonAnalyzer _analyzer; private readonly ILogger _log; private readonly bool _forceGC; - private readonly PathResolverSnapshot _modulesPathResolver; - private readonly PathResolverSnapshot _typeshedPathResolver; - private readonly AsyncCountdownEvent _ace = new AsyncCountdownEvent(0); + private readonly IModuleDatabaseService _moduleDatabaseService; private State _state; private bool _isCanceled; + private int _runningTasks; public bool IsCompleted { get { @@ -82,7 +75,6 @@ public PythonAnalyzerSession(IServiceContainer services, _services = services; _startNextSession = startNextSession; - _analyzerCancellationToken = analyzerCancellationToken; Version = version; AffectedEntriesCount = walker?.AffectedValues.Count ?? 1; @@ -92,24 +84,18 @@ public PythonAnalyzerSession(IServiceContainer services, _forceGC = forceGC; _diagnosticsService = _services.GetService(); - _platformService = _services.GetService(); _analyzer = _services.GetService(); _log = _services.GetService(); + _moduleDatabaseService = _services.GetService(); _progress = progress; - - var interpreter = _services.GetService(); - _modulesPathResolver = interpreter.ModuleResolution.CurrentPathResolver; - _typeshedPathResolver = interpreter.TypeshedResolution.CurrentPathResolver; } public void Start(bool analyzeEntry) { lock (_syncObj) { - if (_state == State.Completed) { - return; - } - if (_state != State.NotStarted) { analyzeEntry = false; + } else if (_state == State.Completed) { + return; } else { _state = State.Started; } @@ -146,26 +132,26 @@ private async Task StartAsync() { try { _log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started."); remaining = await AnalyzeAffectedEntriesAsync(stopWatch); - Debug.Assert(_ace.Count == 0); } finally { stopWatch.Stop(); - var isFinal = false; + bool isCanceled; + bool isFinal; lock (_syncObj) { - if (!_isCanceled) { - _progress.ReportRemaining(remaining); - } - + isCanceled = _isCanceled; _state = State.Completed; isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; _walker = null; } - if (isFinal) { - var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); - totalMilliseconds = Math.Round(totalMilliseconds, 2); - if (await _analyzer.RaiseAnalysisCompleteAsync(modulesCount, totalMilliseconds)) { - _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); + if (!isCanceled) { + _progress.ReportRemaining(remaining); + if (isFinal) { + var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + totalMilliseconds = Math.Round(totalMilliseconds, 2); + if (await _analyzer.RaiseAnalysisCompleteAsync(modulesCount, totalMilliseconds)) { + _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); + } } } } @@ -200,42 +186,43 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai } private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { - IDependencyChainNode node; + IDependencyChainNode node; var remaining = 0; + var ace = new AsyncCountdownEvent(0); + bool isCanceled; while ((node = await _walker.GetNextAsync(_analyzerCancellationToken)) != null) { - var taskLimitReached = false; lock (_syncObj) { - if (_isCanceled) { - switch (node) { - case IDependencyChainLoopNode loop: - // Loop analysis is not cancellable or else small - // inner loops of a larger loop will not be analyzed - // correctly as large loop may cancel inner loop pass. - break; - case IDependencyChainSingleNode single when !single.Value.NotAnalyzed: - remaining++; - node.MoveNext(); - continue; - } - } + isCanceled = _isCanceled; + } + + if (isCanceled && !node.Value.NotAnalyzed) { + remaining++; + node.MoveNext(); + continue; + } - taskLimitReached = _ace.Count >= _maxTaskRunning || _walker.Remaining == 1; - _ace.AddOne(); + ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath); + + var taskLimitReached = false; + lock (_syncObj) { + _runningTasks++; + taskLimitReached = _runningTasks >= _maxTaskRunning || _walker.Remaining == 1; } if (taskLimitReached) { RunAnalysis(node, stopWatch); } else { - StartAnalysis(node, stopWatch).DoNotWait(); + ace.AddOne(); + StartAnalysis(node, ace, stopWatch).DoNotWait(); } } - await _ace.WaitAsync(_analyzerCancellationToken); + await ace.WaitAsync(_analyzerCancellationToken); lock (_syncObj) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { - Debug.Assert(_ace.Count == 0); + Debug.Assert(_runningTasks == 0); } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } @@ -244,240 +231,128 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { return remaining; } - private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) - => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, stopWatch), null); - private Task StartAnalysis(IDependencyChainNode node, Stopwatch stopWatch) - => Task.Run(() => Analyze(node, stopWatch)); + private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node, IDocumentAnalysis currentAnalysis) + => !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion; + + private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) + => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null); + + private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) + => Task.Run(() => Analyze(node, ace, stopWatch)); - private void Analyze(IDependencyChainNode node, Stopwatch stopWatch) { - var loopAnalysis = false; + /// + /// Performs analysis of the document. Returns document global scope + /// with declared variables and inner scopes. Does not analyze chain + /// of dependencies, it is intended for the single file analysis. + /// + private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { try { - switch (node) { - case IDependencyChainSingleNode single: - try { - Analyze(single, stopWatch); - } catch (OperationCanceledException oce) { - single.Value.TryCancel(oce, _walker.Version); - LogCanceled(single.Value.Module); - } catch (Exception exception) { - single.Value.TrySetException(exception, _walker.Version); - node.MarkWalked(); - LogException(single.Value, exception); - } - break; - case IDependencyChainLoopNode loop: - try { - loopAnalysis = true; - AnalyzeLoop(loop, stopWatch); - } catch (OperationCanceledException) { - } catch (Exception exception) { - node.MarkWalked(); - LogException(loop, exception); - } - break; + var entry = node.Value; + if (!CanUpdateAnalysis(entry, node, _walker.Version, out var module, out var ast, out var currentAnalysis)) { + return; } + var startTime = stopWatch.Elapsed; + AnalyzeEntry(node, entry, module, ast, _walker.Version); + + LogCompleted(node, module, stopWatch, startTime); + } catch (OperationCanceledException oce) { + node.Value.TryCancel(oce, _walker.Version); + LogCanceled(node.Value.Module); + } catch (Exception exception) { + node.Value.TrySetException(exception, _walker.Version); + node.MarkWalked(); + LogException(node.Value.Module, exception); } finally { + node.MoveNext(); + lock (_syncObj) { - node.MoveNext(); - if (!_isCanceled || loopAnalysis) { + if (!_isCanceled) { _progress.ReportRemaining(_walker.Remaining); } - _ace.Signal(); + _runningTasks--; + ace?.Signal(); } } } - /// - /// Performs analysis of the document. Returns document global scope - /// with declared variables and inner scopes. Does not analyze chain - /// of dependencies, it is intended for the single file analysis. - /// - private void Analyze(IDependencyChainSingleNode node, Stopwatch stopWatch) { - ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath); - var entry = node.Value; - if (!CanUpdateAnalysis(entry, _walker.Version, out var module, out var ast)) { - return; - } - var startTime = stopWatch.Elapsed; - AnalyzeEntry(node, entry, module, ast, _walker.Version); - - LogCompleted(node, module, stopWatch, startTime); - } - private void AnalyzeEntry() { var stopWatch = _log != null ? Stopwatch.StartNew() : null; try { - if (!CanUpdateAnalysis(_entry, Version, out var module, out var ast)) { + if (!CanUpdateAnalysis(_entry, null, Version, out var module, out var ast, out var currentAnalysis)) { return; } var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; AnalyzeEntry(null, _entry, module, ast, Version); - LogCompleted(module, stopWatch, startTime); + LogCompleted(null, module, stopWatch, startTime); } catch (OperationCanceledException oce) { _entry.TryCancel(oce, Version); LogCanceled(_entry.Module); } catch (Exception exception) { _entry.TrySetException(exception, Version); - LogException(_entry, exception); + LogException(_entry.Module, exception); } finally { stopWatch?.Stop(); } } - private void AnalyzeLoop(IDependencyChainLoopNode loopNode, Stopwatch stopWatch) { - var version = _walker.Version; - var entries = new Dictionary(); - var variables = new Dictionary<(AnalysisModuleKey Module, string Name), int>(); - var importNames = new List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)>(); - var cachedVariables = new Dictionary(); - var asts = new Dictionary(); - var startTime = stopWatch.Elapsed; - - // Note: loop analysis is not cancellable. The reason is that when smaller loop - // appears inside a larger loop gets canceled, it will not be re-walked during - // the outer loop analysis. For example, functools <=> _functools loop and - // the related CircularDependencyFunctools test. - foreach (var entry in loopNode.Values) { - ActivityTracker.OnEnqueueModule(entry.Module.FilePath); - if (!CanUpdateAnalysis(entry, Version, out var module, out var ast)) { - continue; - } - - var moduleKey = new AnalysisModuleKey(module); - entries[moduleKey] = (module, entry); - AddLoopImportsFromAst(importNames, variables, moduleKey, ast); - asts.Add(moduleKey, ast); - } - - if (asts.Count == 0) { - // Fully cached loop - if (_log != null && _log.LogLevel == TraceEventType.Verbose) { - var names = string.Join(", ", cachedVariables.Select(v => v.Key.Name)); - _log?.Log(TraceEventType.Verbose, $"Fully cached modules cycle: {names}"); - } - return; - } - - var imports = new List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, int ToPosition)>(); - foreach (var (fromModule, fromPosition, toModule, toName) in importNames) { - if (!entries.ContainsKey(toModule)) { - continue; - } - - if (toName == null) { - imports.Add((fromModule, fromPosition, toModule, 0)); - } else if (variables.TryGetValue((toModule, toName), out var toPosition)) { - imports.Add((fromModule, fromPosition, toModule, toPosition)); - } - } - - var startingKeys = LocationLoopResolver.FindStartingItems(imports); - var variableHandler = new LoopImportedVariableHandler(_services, asts, cachedVariables, () => false); - foreach (var key in startingKeys) { - if (asts.TryGetValue(key, out var startingAst) && entries.TryGetValue(key, out var me)) { - variableHandler.WalkModule(me.Module, startingAst); - } - } - - foreach (var walker in variableHandler.Walkers) { - asts.Remove(new AnalysisModuleKey(walker.Module)); - } - - while (asts.Count > 0) { - var (moduleKey, ast) = asts.First(); - variableHandler.WalkModule(entries[moduleKey].Module, ast); - - foreach (var walker in variableHandler.Walkers) { - asts.Remove(new AnalysisModuleKey(walker.Module)); - } - } - - foreach (var walker in variableHandler.Walkers) { - var module = (IDocument)walker.Module; - var moduleKey = new AnalysisModuleKey(module); - if (entries.TryGetValue(moduleKey, out var e)) { - var analysis = CreateAnalysis(null, module, walker.Ast, version, walker); - CompleteAnalysis(e.Entry, module, version, analysis); - } - } - - loopNode.MarkWalked(); - LogCompleted(loopNode, entries.Values.Select(v => v.Module), stopWatch, startTime); - } + private bool CanUpdateAnalysis( + PythonAnalyzerEntry entry, + IDependencyChainNode node, + int version, + out IPythonModule module, + out PythonAst ast, + out IDocumentAnalysis currentAnalysis) { - private void AddLoopImportsFromAst( - in List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)> imports, - in Dictionary<(AnalysisModuleKey Module, string Name), int> variables, - in AnalysisModuleKey moduleKey, - in PythonAst ast) { - - var pathResolver = moduleKey.IsTypeshed ? _typeshedPathResolver : _modulesPathResolver; - var walker = new ImportExportWalker(ast, _platformService, pathResolver, moduleKey.FilePath, moduleKey.IsTypeshed); - walker.Walk(); - - foreach (var export in walker.Exports) { - var key = (moduleKey, export.Name); - var location = export.Location.Start; - if (!variables.TryGetValue(key, out var currentLocation) || currentLocation > location) { - variables[key] = location; + if (!entry.CanUpdateAnalysis(version, out module, out ast, out currentAnalysis)) { + if (IsAnalyzedLibraryInLoop(node, currentAnalysis)) { + // Library analysis exists, don't analyze again + return false; } - } - - foreach (var (toModule, name, location) in walker.Imports) { - imports.Add((moduleKey, location.Start, toModule, name)); - } - } - - private bool CanUpdateAnalysis(PythonAnalyzerEntry entry, int version, out IPythonModule module, out PythonAst ast) { - if (entry.CanUpdateAnalysis(version, out module, out ast, out var currentAnalysis)) { - return true; - } - - if (ast == null) { - if (currentAnalysis == null) { - // Entry doesn't have ast yet. There should be at least one more session. - Cancel(); - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet)."); + if (ast == null) { + if (currentAnalysis == null) { + // Entry doesn't have ast yet. There should be at least one more session. + Cancel(); + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet)."); + return false; + } + //Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); return false; } - //Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); + + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}."); return false; } - - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}."); - return false; + return true; } - private void AnalyzeEntry(IDependencyChainSingleNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { + private void AnalyzeEntry(IDependencyChainNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); Debug.Assert(ast != null); - var analysis = AnalyzeModule(node, module, ast, version); + var analysis = DoAnalyzeEntry(node, module, ast, version); _analyzerCancellationToken.ThrowIfCancellationRequested(); if (analysis != null) { - CompleteAnalysis(entry, module, version, analysis); + analyzable?.NotifyAnalysisComplete(analysis); + entry.TrySetAnalysis(analysis, version); + + if (module.ModuleType == ModuleType.User) { + var linterDiagnostics = _analyzer.LintModule(module); + _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); + } } } - private void CompleteAnalysis(PythonAnalyzerEntry entry, IPythonModule module, int version, IDocumentAnalysis analysis) { - var analyzable = module as IAnalyzable; - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); - - if (module.ModuleType != ModuleType.User) { - return; + private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IPythonModule module, PythonAst ast, int version) { + var analysis = TryRestoreCachedAnalysis(node, module); + if (analysis != null) { + return analysis; } - var linterDiagnostics = _analyzer.LintModule(module); - _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); - } - - private IDocumentAnalysis AnalyzeModule(IDependencyChainSingleNode node, IPythonModule module, PythonAst ast, int version) { if (module is IAnalyzable analyzable) { var walker = analyzable.Analyze(ast); return CreateAnalysis(node, (IDocument)module, ast, version, walker); @@ -485,7 +360,38 @@ private IDocumentAnalysis AnalyzeModule(IDependencyChainSingleNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { + private bool MarkNodeWalked(IDependencyChainNode node) { + bool isCanceled; + lock (_syncObj) { + isCanceled = _isCanceled; + } + if (!isCanceled) { + node?.MarkWalked(); + } + return isCanceled; + } + + private IDocumentAnalysis TryRestoreCachedAnalysis(IDependencyChainNode node, IPythonModule module) { + var moduleType = module.ModuleType; + if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(module.Name, module.FilePath) == true) { + if (_moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, "Restored from database: ", module.Name); + } + var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty()); + gs.ReconstructVariables(); + MarkNodeWalked(node); + return analysis; + } else { + if (_log != null) { + _log.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name); + } + } + } + return null; + } + + private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { var canHaveLibraryAnalysis = false; // Don't try to drop builtins; it causes issues elsewhere. @@ -494,73 +400,48 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode(); - if (optionsProvider?.Options.KeepLibraryAst == true) { - createLibraryAnalysis = false; - } + var optionsProvider = _services.GetService(); + if (optionsProvider?.Options.KeepLibraryAst == true) { + createLibraryAnalysis = false; + } - if (!createLibraryAnalysis) { - return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); - } + if (!createLibraryAnalysis) { + return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + } - if (document.ModuleType != ModuleType.Stub && !_isCanceled) { - ast.ReduceToImports(); - document.SetAst(ast); - } + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); + document.SetAst(ast); - var eval = new ExpressionEval(walker.Eval.Services, document, ast); - var analysis = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + var eval = new ExpressionEval(walker.Eval.Services, document, ast); + var analysis = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); - var dbs = _services.GetService(); - dbs?.StoreModuleAnalysisAsync(analysis, immediate: false, _analyzerCancellationToken).DoNotWait(); + var dbs = _services.GetService(); + dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait(); - return analysis; - } + return analysis; } - private void LogCompleted(IDependencyChainLoopNode node, IEnumerable modules, Stopwatch stopWatch, TimeSpan startTime) { - if (_log != null) { - var moduleNames = modules.Select(m => "{0}({1})".FormatInvariant(m.Name, m.Analysis is LibraryAnalysis ? "Library" : m.ModuleType.ToString())); - var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); - var message = $"Analysis of modules loop on depth {node.VertexDepth} in {elapsed} ms:"; - _log.Log(TraceEventType.Verbose, message); - foreach (var name in moduleNames) { - _log.Log(TraceEventType.Verbose, $" {name}"); - } - } - } - private void LogCompleted(IDependencyChainSingleNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { if (_log != null) { - var completed = module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; + var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); - var message = $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {elapsed} ms."; - _log.Log(TraceEventType.Verbose, message); - } - } - - private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { - if (_log != null) { - var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); - var message = $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {elapsed} ms."; + var message = node != null + ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {elapsed} ms." + : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {elapsed} ms."; _log.Log(TraceEventType.Verbose, message); } } @@ -571,21 +452,9 @@ private void LogCanceled(IPythonModule module) { } } - private void LogException(PythonAnalyzerEntry entry, Exception exception) { + private void LogException(IPythonModule module, Exception exception) { if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) failed. {exception}"); - } - - if (TestEnvironment.Current != null) { - ExceptionDispatchInfo.Capture(exception).Throw(); - } - } - - private void LogException(IDependencyChainLoopNode node, Exception exception) { - if (_log != null) { - var moduleNames = string.Join(", ", node.Values.Select(e => $"{e.Module.Name}({e.Module.ModuleType})")); - var message = $"Analysis of modules loop [{moduleNames}] failed. {exception}"; - _log.Log(TraceEventType.Verbose, message); + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}"); } if (TestEnvironment.Current != null) { diff --git a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs index 2636a5691..188d41c0a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs +++ b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs @@ -75,7 +75,6 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT if (stubType.IsUnknown()) { continue; } - // If stub says 'Any' but we have better type, keep the current type. if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { continue; @@ -85,7 +84,7 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT var sourceType = sourceVar?.Value.GetPythonType(); if (sourceVar?.Source == VariableSource.Import && - sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { + sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { // Keep imported types as they are defined in the library. For example, // 'requests' imports NullHandler as 'from logging import NullHandler'. // But 'requests' also declares NullHandler in its stub (but not in the main code) @@ -298,7 +297,7 @@ private void TransferDocumentationAndLocation(IPythonType sourceType, IPythonTyp // Consider that 'email.headregistry' stub has DataHeader declaring 'datetime' // property of type 'datetime' from 'datetime' module. We don't want to modify // datetime type and change it's location to 'email.headregistry'. - if (stubType.DeclaringModule.ModuleType != ModuleType.Stub || stubType.DeclaringModule != _eval.Module.Stub) { + if(stubType.DeclaringModule.ModuleType != ModuleType.Stub || stubType.DeclaringModule != _eval.Module.Stub) { return; } @@ -348,7 +347,7 @@ private void TransferDocumentationAndLocation(IPythonType sourceType, IPythonTyp /// or location of unrelated types such as coming from the base object type. /// private bool IsFromThisModuleOrSubmodules(IPythonType type) { - if (type.IsUnknown()) { + if(type.IsUnknown()) { return false; } var thisModule = _eval.Module; diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 39c7b128a..5151a1a68 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -73,7 +73,7 @@ public override void Evaluate() { v => v.GetPythonType() == null && v.GetPythonType() == null) ) { - ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); } } } @@ -99,7 +99,6 @@ public static IMember GetReturnValueFromAnnotation(ExpressionEval eval, Expressi var instance = t.IsUnknown() ? (IMember)annotationType : t; return instance; } - private IMember TryDetermineReturnValue() { var returnType = GetReturnValueFromAnnotation(Eval, FunctionDefinition.ReturnAnnotation); if (returnType != null) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs index 6cc3c00f3..b2f63639c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs @@ -16,14 +16,13 @@ using System; using System.Diagnostics; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Symbols { [DebuggerDisplay("{Target.Name}")] internal abstract class MemberEvaluator : AnalysisWalker { - protected MemberEvaluator(ExpressionEval eval, ScopeStatement target) : base(eval, SimpleImportedVariableHandler.Instance) { + protected MemberEvaluator(ExpressionEval eval, ScopeStatement target) : base(eval) { Target = target ?? throw new ArgumentNullException(nameof(target)); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index b383758d1..07c1afb18 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -112,6 +112,7 @@ private PythonClassType CreateClass(ClassDefinition cd) { return cls; } + private void AddFunctionOrProperty(FunctionDefinition fd) { var declaringType = fd.Parent != null && _typeMap.TryGetValue(fd.Parent, out var t) ? t : null; if (!TryAddProperty(fd, declaringType)) { @@ -135,7 +136,7 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { AddOverload(fd, f, o => f.AddOverload(o)); } - private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { + private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { // Check if function exists in stubs. If so, take overload from stub // and the documentation from this actual module. if (!_table.ReplacedByStubs.Contains(fd)) { @@ -165,7 +166,7 @@ private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { if (t is IPythonFunctionType f) { return f.Overloads .OfType() - .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Count(p => !p.IsPositionalOnlyMarker)); + .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Where(p => !p.IsPositionalOnlyMarker).Count()); } return null; } diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index e04965a97..fb40a5bfc 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -15,36 +15,34 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Caching { internal interface IModuleDatabaseService: IModuleDatabaseCache { /// - /// Restores module from database. + /// Creates global scope from module persistent state. + /// Global scope is then can be used to construct module analysis. /// - IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType); + /// Python module to restore analysis for. + /// Python module global scope. + bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs); + + /// + /// Retrieves dependencies from the module persistent state. + /// + /// Python module to restore analysis for. + /// Python module dependency provider. + bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp); /// /// Writes module data to the database. /// - /// Document analysis - /// - /// True if database should be written to disk immediately - /// as opposed to delaying writing until complete analysis event from the - /// - /// Cancellation token - Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default); + Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default); /// /// Determines if module analysis exists in the storage. /// - bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType); - } - - internal static class ModuleDatabaseExtensions { - public static bool ModuleExistsInStorage(this IModuleDatabaseService dbs, IPythonModule module) - => dbs.ModuleExistsInStorage(module.Name, module.FilePath, module.ModuleType); + bool ModuleExistsInStorage(string moduleName, string filePath); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs similarity index 60% rename from src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs rename to src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs index b269186d6..71bf8e130 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs @@ -1,4 +1,4 @@ -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -13,14 +13,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; -using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; -namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal interface IImportedVariableHandler { - IEnumerable GetMemberNames(PythonVariableModule variableModule); - IVariable GetVariable(in PythonVariableModule module, in string name); - void EnsureModule(in PythonVariableModule module); +namespace Microsoft.Python.Analysis.Caching { + /// + /// Represents global scope that has been restored from + /// the database but has not been fully populated yet. + /// Used to attach to analysis so variables can be + /// accessed during classes and methods restoration. + /// + internal interface IRestoredGlobalScope : IGlobalScope { + void ReconstructVariables(); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs index 381fc40f9..a0c3e4cca 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs @@ -29,9 +29,9 @@ internal sealed class DependencyCollector { public ISet Dependencies { get; } = new HashSet(); - public DependencyCollector(IPythonModule module, bool? isTypeshed = null) { + public DependencyCollector(IPythonModule module, bool? isTypeShed = null) { _module = module; - _isTypeshed = isTypeshed ?? module.IsTypeshed; + _isTypeshed = isTypeShed ?? module.IsTypeshed; _moduleResolution = module.Interpreter.ModuleResolution; _pathResolver = _isTypeshed ? module.Interpreter.TypeshedResolution.CurrentPathResolver diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index ca8be84fc..0813a6303 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -102,28 +101,51 @@ public int Remove(in TKey key) { } } - /// - /// Removes everything but builtins. - /// - public void Reset() { + public int RemoveKeys(params TKey[] keys) => RemoveKeys(ImmutableArray.Create(keys)); + + public int RemoveKeys(in ImmutableArray keys) { lock (_syncObj) { - if (_vertices.Count > 1) { - _vertices.RemoveRange(1, _vertices.Count - 1); + foreach (var key in keys) { + if (_keys.TryGetValue(key, out var index)) { + _vertices[index] = default; + } } - var kvp = _keys.Count > 0 ? _keys.FirstOrDefault(k => k.Value == 0) : (KeyValuePair?)null; + var oldKeysReversed = _keys.ToDictionary(kvp => kvp.Value, kvp => kvp.Key); + var oldVertices = new DependencyVertex[_vertices.Count]; + _vertices.CopyTo(oldVertices); + _keys.Clear(); - if(kvp != null) { - _keys[kvp.Value.Key] = 0; + _vertices.Clear(); + + foreach (var oldVertex in oldVertices) { + if (oldVertex == null) { + continue; + } + + var incomingKeys = oldVertex.Incoming.Select(i => oldKeysReversed[i]); + var key = oldVertex.Key; + var value = oldVertex.Value; + var isRoot = oldVertex.IsRoot; + + if (!_keys.TryGetValue(key, out var index)) { + index = _keys.Count; + _keys[key] = index; + _vertices.Add(default); + } + + Update(key, value, isRoot, incomingKeys, index); } - _version++; + return _version; } } private void Update(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys, in int index) { var version = Interlocked.Increment(ref _version); + var incoming = EnsureKeys(index, incomingKeys, version); + _vertices[index] = new DependencyVertex(key, value, isRoot, incoming, version, index); _keys[key] = index; } @@ -185,7 +207,7 @@ public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDepend return false; } - if (!TryResolveLoops(walkingGraph, loopsCount, version, out var loopNodes)) { + if (!TryResolveLoops(walkingGraph, loopsCount, version, out var totalNodesCount)) { walker = default; return false; } @@ -196,15 +218,14 @@ public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDepend return false; } - var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); - - walkingGraph = walkingGraph.AddRange(loopNodes); foreach (var vertex in walkingGraph) { vertex.Seal(); + vertex.SecondPass?.Seal(); } + var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); var startingVertices = walkingGraph.Where(v => !v.HasIncoming); - walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, version); + walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); return version == _version; } @@ -222,7 +243,11 @@ private bool TryBuildReverseGraph(in ImmutableArray[vertices.Count]; - foreach (var vertex in vertices.Where(vertex => vertex != null)) { + foreach (var vertex in vertices) { + if (vertex == null) { + continue; + } + if (version != _version) { return false; } @@ -245,8 +270,10 @@ private bool TryBuildReverseGraph(in ImmutableArray vertex != null && !vertex.IsSealed)) { - vertex.Seal(outgoingVertices[vertex.Index]); + foreach (var vertex in vertices) { + if (vertex != null && !vertex.IsSealed) { + vertex.Seal(outgoingVertices[vertex.Index]); + } } return true; @@ -407,62 +434,109 @@ private static bool SetLoopNumber(WalkingVertex vertex, Stack> graph, int loopsCount, int version, out ImmutableArray> loopVertices) { - loopVertices = ImmutableArray>.Empty; + private bool TryResolveLoops(in ImmutableArray> graph, int loopsCount, int version, out int totalNodesCount) { if (loopsCount == 0) { + totalNodesCount = graph.Count; return true; } - // Create independent walking vertices for vertex loops - for (var i = 0; i < loopsCount; i++) { - loopVertices = loopVertices.Add(new WalkingVertex(i)); - } - - // Break internal loop connections + // Create vertices for second pass + var inLoopsCount = 0; + var secondPassLoops = new List>[loopsCount]; foreach (var vertex in graph) { if (vertex.IsInLoop) { + var secondPassVertex = vertex.CreateSecondPassVertex(); var loopNumber = vertex.LoopNumber; - var loopVertex = loopVertices[loopNumber]; - - for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { - if (vertex.Outgoing[i].LoopNumber == loopNumber) { - vertex.RemoveOutgoingAt(i); - } + if (secondPassLoops[loopNumber] == null) { + secondPassLoops[loopNumber] = new List> { secondPassVertex }; + } else { + secondPassLoops[loopNumber].Add(secondPassVertex); } - loopVertex.AddOutgoing(vertex); + inLoopsCount++; } if (version != _version) { + totalNodesCount = default; return false; } + + vertex.Index = -1; // Reset index, will use later } - // Connect dependencies to loop vertex - var outgoingLoopVertices = new HashSet>(); - foreach (var vertex in graph) { - outgoingLoopVertices.Clear(); - for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { - var outgoing = vertex.Outgoing[i]; - if (outgoing.IsInLoop && outgoing.LoopNumber != vertex.LoopNumber) { - var loopVertex = loopVertices[outgoing.LoopNumber]; - vertex.RemoveOutgoingAt(i); - outgoingLoopVertices.Add(loopVertex); + // Break the loops so that its items can be iterated + foreach (var loop in secondPassLoops) { + // Sort loop items by amount of incoming connections + loop.Sort(WalkingVertex.FirstPassIncomingComparison); + + var counter = 0; + foreach (var secondPassVertex in loop) { + var vertex = secondPassVertex.FirstPass; + if (vertex.Index == -1) { + RemoveOutgoingLoopEdges(vertex, ref counter); + } + + if (version != _version) { + totalNodesCount = default; + return false; } } + } + + // Make first vertex from second pass loop (loop is sorted at this point) have incoming edges from vertices from first pass loop and set unique loop numbers + var outgoingVertices = new HashSet>(); + foreach (var loop in secondPassLoops) { + outgoingVertices.Clear(); + var startVertex = loop[0]; - if (outgoingLoopVertices.Count > 0) { - vertex.AddOutgoing(outgoingLoopVertices); + foreach (var secondPassVertex in loop) { + var firstPassVertex = secondPassVertex.FirstPass; + firstPassVertex.AddOutgoing(startVertex); + + foreach (var outgoingVertex in firstPassVertex.Outgoing) { + if (outgoingVertex.LoopNumber != firstPassVertex.LoopNumber) { + // Collect outgoing vertices to reference them from loop + outgoingVertices.Add(outgoingVertex); + } else if (outgoingVertex.SecondPass != null) { + // Copy outgoing edges to the second pass vertex + secondPassVertex.AddOutgoing(outgoingVertex.SecondPass); + } + } + } + + // Add outgoing edges to all second pass vertices to ensure that further analysis won't start until loop is fully analyzed + foreach (var secondPassVertex in loop) { + secondPassVertex.AddOutgoing(outgoingVertices); } if (version != _version) { + totalNodesCount = default; return false; } + + loopsCount++; } + totalNodesCount = graph.Count + inLoopsCount; return true; } + private static void RemoveOutgoingLoopEdges(WalkingVertex vertex, ref int counter) { + vertex.Index = counter++; + for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { + var outgoing = vertex.Outgoing[i]; + if (outgoing.LoopNumber != vertex.LoopNumber) { + continue; + } + + if (outgoing.Index == -1) { + RemoveOutgoingLoopEdges(outgoing, ref counter); + } else if (outgoing.Index < vertex.Index) { + vertex.RemoveOutgoingAt(i); + } + } + } + private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version, out ImmutableArray missingKeys) { var haveMissingDependencies = new bool[vertices.Count]; var queue = new Queue>(); @@ -507,8 +581,9 @@ private bool TryFindMissingDependencies(in ImmutableArray _dependencyResolver; private readonly ImmutableArray> _startingVertices; private readonly ImmutableArray _depths; - private readonly object _syncObj = new object(); + private readonly object _syncObj; private int _remaining; - private PriorityProducerConsumer _ppc; + private PriorityProducerConsumer> _ppc; public ImmutableArray MissingKeys { get; set; } public ImmutableArray AffectedValues { get; } @@ -548,8 +623,10 @@ public DependencyChainWalker(in DependencyResolver dependencyResol in ImmutableArray affectedValues, in ImmutableArray depths, in ImmutableArray missingKeys, + in int totalNodesCount, in int version) { + _syncObj = new object(); _dependencyResolver = dependencyResolver; _startingVertices = startingVertices; _depths = depths; @@ -557,17 +634,17 @@ public DependencyChainWalker(in DependencyResolver dependencyResol Version = version; MissingKeys = missingKeys; - _remaining = affectedValues.Count; + _remaining = totalNodesCount; } - public Task GetNextAsync(CancellationToken cancellationToken) { - PriorityProducerConsumer ppc; + public Task> GetNextAsync(CancellationToken cancellationToken) { + PriorityProducerConsumer> ppc; lock (_syncObj) { if (_ppc == null) { - _ppc = new PriorityProducerConsumer(); + _ppc = new PriorityProducerConsumer>(); foreach (var vertex in _startingVertices) { - _ppc.Produce(CreateNode(vertex)); + _ppc.Produce(new DependencyChainNode(this, vertex, _depths[vertex.DependencyVertex.Index])); } } @@ -577,7 +654,7 @@ public Task GetNextAsync(CancellationToken cancellationTok return ppc.ConsumeAsync(cancellationToken); } - public void MoveNext(WalkingVertex vertex, bool loopAnalysis) { + public void MoveNext(WalkingVertex vertex) { var verticesToProduce = new List>(); var isCompleted = false; lock (_syncObj) { @@ -587,7 +664,7 @@ public void MoveNext(WalkingVertex vertex, bool loopAnalysis) { continue; } - outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming || loopAnalysis); + outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming); if (outgoing.HasIncoming) { continue; } @@ -604,7 +681,7 @@ public void MoveNext(WalkingVertex vertex, bool loopAnalysis) { _ppc.Produce(null); } else { foreach (var toProduce in verticesToProduce) { - _ppc.Produce(CreateNode(toProduce)); + _ppc.Produce(new DependencyChainNode(this, toProduce, _depths[toProduce.DependencyVertex.Index])); } } } @@ -616,68 +693,31 @@ public bool IsValidVersion { } } } - - private IDependencyChainNode CreateNode(WalkingVertex vertex) { - if (vertex.DependencyVertex != null) { - return new SingleNode(this, vertex, _depths[vertex.DependencyVertex.Index]); - } - - var vertices = vertex.Outgoing; - var values = vertices.Select(v => v.DependencyVertex.Value).ToImmutableArray(); - var depth = vertices.Min(v => _depths[v.DependencyVertex.Index]); - var hasMissingDependencies = vertices.Any(v => v.HasMissingDependencies); - return new LoopNode(this, vertices, values, depth, hasMissingDependencies); - } } - [DebuggerDisplay("{" + nameof(Value) + "}")] - private sealed class SingleNode : IDependencyChainSingleNode { + private sealed class DependencyChainNode : IDependencyChainNode { private readonly WalkingVertex _vertex; private DependencyChainWalker _walker; public TValue Value => _vertex.DependencyVertex.Value; public int VertexDepth { get; } public bool HasMissingDependencies => _vertex.HasMissingDependencies; - public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming; + public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.SecondPass == null; public bool IsWalkedWithDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.DependencyVertex.IsWalked; public bool IsValidVersion => _walker.IsValidVersion; - public SingleNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) => (_walker, _vertex, VertexDepth) = (walker, vertex, depth); - - public void MarkWalked() => _vertex.DependencyVertex.MarkWalked(); - - public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex, loopAnalysis: false); - } - - [DebuggerDisplay("Loop: {_vertices.Count} nodes")] - private sealed class LoopNode : IDependencyChainLoopNode { - private readonly IReadOnlyList> _vertices; - private DependencyChainWalker _walker; - - public int VertexDepth { get; } - public bool HasMissingDependencies { get; } - public bool HasOnlyWalkedDependencies => _vertices.All(v => v.HasOnlyWalkedIncoming); - public bool IsWalkedWithDependencies => _vertices.All(v => v.HasOnlyWalkedIncoming && v.DependencyVertex.IsWalked); - public bool IsValidVersion => _walker.IsValidVersion; - - public ImmutableArray Values { get; } - - public LoopNode(DependencyChainWalker walker, IReadOnlyList> vertices, ImmutableArray values, int depth, bool hasMissingDependencies) - => (_walker, _vertices, Values, VertexDepth, HasMissingDependencies) = (walker, vertices, values, depth, hasMissingDependencies); + public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { + _walker = walker; + _vertex = vertex; + VertexDepth = depth; + } public void MarkWalked() { - foreach (var vertex in _vertices) { - vertex.DependencyVertex.MarkWalked(); + if (_vertex.SecondPass == null) { + _vertex.DependencyVertex.MarkWalked(); } } - public void MoveNext() { - var walker = Interlocked.Exchange(ref _walker, null); - if (walker != null) { - foreach (var vertex in _vertices) { - walker.MoveNext(vertex, loopAnalysis: true); - } - } - } + public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex); } } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 09c1a1cf5..ddbf9160d 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -36,7 +36,7 @@ internal sealed class DependencyVertex { private int _state; private HashSet _outgoing; - private static readonly HashSet _empty = new HashSet(); + private static HashSet _empty = new HashSet(); public DependencyVertex(DependencyVertex oldVertex, int version, bool isNew) { Key = oldVertex.Key; diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs index f0ef93f98..2f2270251 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index 2532cb6e6..a7fca9a45 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -13,10 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Core.Collections; - namespace Microsoft.Python.Analysis.Dependencies { - internal interface IDependencyChainNode { + internal interface IDependencyChainNode { int VertexDepth { get; } /// @@ -35,15 +33,8 @@ internal interface IDependencyChainNode { /// Returns true if node version matches version of the walked graph /// bool IsValidVersion { get; } + TValue Value { get; } void MarkWalked(); void MoveNext(); } - - internal interface IDependencyChainSingleNode : IDependencyChainNode { - TValue Value { get; } - } - - internal interface IDependencyChainLoopNode : IDependencyChainNode { - ImmutableArray Values { get; } - } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs index 4a34f0087..324aadc5f 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs @@ -23,6 +23,6 @@ internal interface IDependencyChainWalker { ImmutableArray AffectedValues { get; } int Version { get; } int Remaining { get; } - Task GetNextAsync(CancellationToken cancellationToken); + Task> GetNextAsync(CancellationToken cancellationToken); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index 203ea5a71..3816803e3 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -26,7 +26,7 @@ internal interface IDependencyResolver { int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); int Remove(in TKey key); - void Reset(); + int RemoveKeys(in ImmutableArray keys); IDependencyChainWalker CreateWalker(); bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker); diff --git a/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs b/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs deleted file mode 100644 index 4710d0ff8..000000000 --- a/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Collections; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Core.OS; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Dependencies { - internal sealed class ImportExportWalker : PythonWalker { - private readonly Dictionary<(AnalysisModuleKey Module, string Name), IndexSpan> _imports; - private readonly Dictionary _exports; - private readonly PythonAst _ast; - private readonly IOSPlatform _platformService; - private readonly PathResolverSnapshot _pathResolver; - private readonly string _filePath; - private readonly bool _isTypeshed; - private int _depth; - - public IEnumerable<(AnalysisModuleKey Module, string Name, IndexSpan Location)> Imports - => _imports.Select(kvp => (kvp.Key.Module, kvp.Key.Name, kvp.Value)); - public IEnumerable<(string Name, IndexSpan Location)> Exports - => _exports.Select(kvp => (kvp.Key, kvp.Value)); - - public ImportExportWalker(PythonAst ast, IOSPlatform platformService, PathResolverSnapshot pathResolver, string filePath, bool isTypeshed) { - _imports = new Dictionary<(AnalysisModuleKey Module, string Name), IndexSpan>(); - _exports = new Dictionary(); - _ast = ast; - _platformService = platformService; - _pathResolver = pathResolver; - _isTypeshed = isTypeshed; - _filePath = filePath; - } - - public void Walk() => _ast.Walk(this); - - public override bool Walk(AssignmentStatement node) { - if (_depth == 0) { - HandleAssignment(node); - } - return base.Walk(node); - } - - private void HandleAssignment(AssignmentStatement node) { - foreach (var expr in node.Left.Select(s => s.RemoveParenthesis()).OfType()) { - AddExport(expr.Name, expr.IndexSpan); - } - - if (node.Right is MemberExpression me) { - AddImportIfModule(me); - } - } - - public override bool Walk(IfStatement node) => node.WalkIfWithSystemConditions(this, _ast.LanguageVersion, _platformService); - - public override bool Walk(ImportStatement expr) { - if (_depth > 0) { - return false; - } - - var len = Math.Min(expr.Names.Count, expr.AsNames.Count); - var forceAbsolute = expr.ForceAbsolute; - for (var i = 0; i < len; i++) { - var moduleImportExpression = expr.Names[i]; - var asNameExpression = expr.AsNames[i]; - - if (!string.IsNullOrEmpty(asNameExpression?.Name)) { - AddLastModuleImport(moduleImportExpression, asNameExpression, forceAbsolute); - } else { - AddAllImports(moduleImportExpression, forceAbsolute); - } - } - - return false; - } - - private void AddLastModuleImport(ModuleName importExpression, NameExpression importName, bool forceAbsolute) { - var result = _pathResolver.GetImportsFromAbsoluteName(_filePath, importExpression.Names.Select(n => n.Name), forceAbsolute); - if (result is ModuleImport mi) { - AddImport(mi, default, importName.IndexSpan); - } - } - - private void AddAllImports(ModuleName moduleImportExpression, bool forceAbsolute) { - var importNames = ImmutableArray.Empty; - - for (var i = 0; i < moduleImportExpression.Names.Count; i++) { - var nameExpression = moduleImportExpression.Names[i]; - importNames = importNames.Add(nameExpression.Name); - var result = _pathResolver.GetImportsFromAbsoluteName(_filePath, importNames, forceAbsolute); - if (result is ModuleImport mi && !mi.ModulePath.PathEquals(_filePath)) { - AddImport(mi, default, nameExpression.IndexSpan); - if (i == 0) { - AddExport(nameExpression.Name, nameExpression.IndexSpan); - } - } - } - } - - public override bool Walk(FromImportStatement expr) { - if (_depth > 0) { - return base.Walk(expr); - } - - var rootNames = expr.Root.Names; - if (rootNames.Count == 1 && rootNames[0].Name.EqualsOrdinal("__future__")) { - return base.Walk(expr); - } - - var imports = _pathResolver.FindImports(_filePath, expr); - if (!(imports is ModuleImport mi)) { - return base.Walk(expr); - } - - var names = expr.Names; - var asNames = expr.AsNames; - if (names.Count == 1 && names[0].Name == "*") { - AddImport(mi, default, names[0].IndexSpan); - return base.Walk(expr); - } - - for (var i = 0; i < names.Count; i++) { - var memberName = names[i].Name; - if (string.IsNullOrEmpty(memberName)) { - continue; - } - - var nameExpression = asNames[i] ?? names[i]; - if (mi.TryGetChildImport(nameExpression.Name, out var child) && child is ModuleImport childMi) { - AddImport(childMi, default, nameExpression.IndexSpan); - } else { - AddImport(mi, names[i].Name, nameExpression.IndexSpan); - } - - AddExport(nameExpression.Name, nameExpression.IndexSpan); - } - - return base.Walk(expr); - } - - public override bool Walk(MemberExpression expr) { - if (_depth == 0) { - AddImportIfModule(expr); - } - - return base.Walk(expr); - } - - public override bool Walk(ClassDefinition cd) { - if (_depth == 0 && !string.IsNullOrEmpty(cd.Name)) { - AddExport(cd.Name, cd.NameExpression.IndexSpan); - } - _depth++; - return base.Walk(cd); - } - - public override void PostWalk(ClassDefinition cd) { - _depth--; - base.PostWalk(cd); - } - - public override bool Walk(FunctionDefinition fd) { - if (_depth == 0 && !string.IsNullOrEmpty(fd.Name)) { - AddExport(fd.Name, fd.NameExpression.IndexSpan); - } - _depth++; - return base.Walk(fd); - } - - public override void PostWalk(FunctionDefinition fd) { - _depth--; - base.PostWalk(fd); - } - - private void AddExport(in string name, IndexSpan location) { - if (!_exports.TryGetValue(name, out var current) || current.Start > location.Start) { - _exports[name] = location; - } - } - - private void AddImportIfModule(in MemberExpression expr) { - var currentExpression = expr; - var memberExpressions = new Stack(); - memberExpressions.Push(currentExpression); - - while (currentExpression.Target is MemberExpression me) { - memberExpressions.Push(me); - currentExpression = me; - } - - if (!(currentExpression.Target is NameExpression ne)) { - return; - } - - var import = _pathResolver.GetModuleImportFromModuleName(ne.Name); - if (import == null) { - return; - } - - var moduleKey = new AnalysisModuleKey(import.Name, import.ModulePath, _isTypeshed); - IImportChildrenSource childrenSource = _pathResolver.GetModuleImportFromModuleName(moduleKey.Name); - if (childrenSource == null) { - return; - } - - while (memberExpressions.Count > 0) { - var expression = memberExpressions.Pop(); - - if (!childrenSource.TryGetChildImport(expression.Name, out var child)) { - AddImport(moduleKey, expression.Name, expression.IndexSpan); - return; - } - - if (child is IImportChildrenSource cs) { - childrenSource = cs; - } else { - return; - } - } - } - - private void AddImport(in ModuleImport moduleImport, in string name, in IndexSpan location) - => AddImport(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed), name, location); - - private void AddImport(in AnalysisModuleKey key, in string name, in IndexSpan location) { - if (key.FilePath.PathEquals(_filePath)) { - return; - } - - if (_imports.TryGetValue((key, name), out var current) && current.Start <= location.Start) { - return; - } - - _imports[(key, name)] = location; - } - } -} diff --git a/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs b/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs deleted file mode 100644 index 8ce8bc6c2..000000000 --- a/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Collections; - -namespace Microsoft.Python.Analysis.Dependencies { - internal static class LocationLoopResolver { - public static ImmutableArray FindStartingItems(IEnumerable<(T From, int FromLocation, T To, int ToLocation)> edges) { - var itemToIndex = new Dictionary(); - var groupedEdges = new List>(); - var index = 0; - foreach (var (fromItem, fromLocation, toItem, toLocation) in edges) { - if (!itemToIndex.TryGetValue(fromItem, out var fromIndex)) { - fromIndex = index++; - groupedEdges.Add(new List<(int, int, int)>()); - itemToIndex[fromItem] = fromIndex; - } - - if (!itemToIndex.TryGetValue(toItem, out var toIndex)) { - toIndex = index++; - groupedEdges.Add(new List<(int, int, int)>()); - itemToIndex[toItem] = toIndex; - } - - groupedEdges[fromIndex].Add((fromLocation, toIndex, toLocation)); - } - - foreach (var group in groupedEdges) { - group.Sort(SortByFromLocation); - } - - var startingIndices = FindStartingIndices(groupedEdges); - return startingIndices.Select(i => itemToIndex.First(j => j.Value == i).Key).ToImmutableArray(); - - int SortByFromLocation((int FromLocation, int, int) x, (int FromLocation, int, int) y) => x.FromLocation.CompareTo(y.FromLocation); - } - - private static IEnumerable FindStartingIndices(List> groupedEdges) { - var walkedIndices = new int[groupedEdges.Count]; - var visited = new bool[groupedEdges.Count]; - var path = new Stack(); - var startingIndex = 0; - var allVisitedBeforeIndex = 0; - - while (startingIndex < groupedEdges.Count) { - if (visited[startingIndex]) { - if (startingIndex == allVisitedBeforeIndex) { - allVisitedBeforeIndex++; - } - - startingIndex++; - continue; - } - - for (var i = 0; i < walkedIndices.Length; i++) { - walkedIndices[i] = -1; - } - - path.Clear(); - - if (!IsWalkable(groupedEdges, startingIndex, walkedIndices, visited, path)) { - startingIndex++; - continue; - } - - for (var i = 0; i < walkedIndices.Length; i++) { - if (walkedIndices[i] != -1) { - visited[i] = true; - } - } - - yield return startingIndex; - startingIndex = allVisitedBeforeIndex; - } - } - - private static bool IsWalkable(in List> groupedEdges, in int startGroupIndex, in int[] walkedIndices, in bool[] visited, in Stack path) { - const int notVisited = -1; - var fromGroupIndex = startGroupIndex; - - while (true) { - var indexInFromGroup = ++walkedIndices[fromGroupIndex]; - var fromGroup = groupedEdges[fromGroupIndex]; - if (fromGroup.Count == indexInFromGroup) { - if (path.Count == 0) { - return true; - } - - fromGroupIndex = path.Pop(); - continue; - } - - var edge = fromGroup[indexInFromGroup]; - var toGroupIndex = edge.ToIndex; - if (visited[toGroupIndex]) { - continue; - } - - var indexInToGroup = walkedIndices[toGroupIndex]; - if (indexInToGroup == notVisited) { - path.Push(fromGroupIndex); - fromGroupIndex = toGroupIndex; - continue; - } - - var toGroup = groupedEdges[toGroupIndex]; - if (toGroup.Count == indexInToGroup) { - continue; - } - - var requiredPosition = edge.ToLocation; - var currentPosition = toGroup[indexInToGroup].FromLocation; - if (requiredPosition > currentPosition) { - return false; - } - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs index 041709891..d3debee4f 100644 --- a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Diagnostics; using Microsoft.Python.Core.Diagnostics; @@ -20,6 +21,8 @@ namespace Microsoft.Python.Analysis.Dependencies { [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] internal sealed class WalkingVertex { + public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass._incomingCount.CompareTo(v2.FirstPass._incomingCount); + private readonly List> _outgoing; private bool _isSealed; private int _incomingCount; @@ -33,21 +36,18 @@ internal sealed class WalkingVertex { public bool HasOnlyWalkedIncoming => _walkedIncomingCount == 0; public bool HasMissingDependencies { get; private set; } + public WalkingVertex FirstPass { get; } + public WalkingVertex SecondPass { get; private set; } + public bool IsInLoop => LoopNumber >= 0; - public string DebuggerDisplay => DependencyVertex?.DebuggerDisplay ?? "Loop node"; + public string DebuggerDisplay => DependencyVertex.DebuggerDisplay; - public WalkingVertex(DependencyVertex vertex) { + public WalkingVertex(DependencyVertex vertex, WalkingVertex firstPass = null) { DependencyVertex = vertex; + FirstPass = firstPass; Index = -1; - LoopNumber = -1; - _outgoing = new List>(); - } - - public WalkingVertex(int loopNumber) { - DependencyVertex = default; - Index = -1; - LoopNumber = loopNumber; + LoopNumber = firstPass?.LoopNumber ?? -1; _outgoing = new List>(); } @@ -83,6 +83,13 @@ public void RemoveOutgoingAt(int index) { outgoingVertex._walkedIncomingCount--; } + public WalkingVertex CreateSecondPassVertex() { + CheckNotSealed(); + + SecondPass = new WalkingVertex(DependencyVertex, this); + return SecondPass; + } + public void Seal() => _isSealed = true; public void DecrementIncoming(bool isWalkedIncoming) { diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 0bd2136a2..9b2e259a0 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -85,7 +85,6 @@ public int DocumentCount { /// Optional file path, if different from the URI. public IDocument OpenDocument(Uri uri, string content, string filePath = null) { bool justOpened; - var created = false; IDocument document; lock (_lock) { var entry = FindDocument(uri); @@ -106,16 +105,11 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { ModuleType = moduleType }; entry = CreateDocument(mco); - created = true; } justOpened = TryOpenDocument(entry, content); document = entry.Document; } - if (created) { - _services.GetService().InvalidateAnalysis(document); - } - if (justOpened) { Opened?.Invoke(this, new DocumentEventArgs(document)); } @@ -247,10 +241,10 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) { IDocument document; switch (mco.ModuleType) { case ModuleType.Compiled when TryAddModulePath(mco): - document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsTypeshed, _services); + document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsPersistent, mco.IsTypeshed, _services); break; case ModuleType.CompiledBuiltin: - document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, _services); + document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, mco.IsPersistent, _services); break; case ModuleType.User: TryAddModulePath(mco); diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index e5ce15f5a..f3ed567cb 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -13,11 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Text; @@ -25,35 +22,6 @@ namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { - public static IEnumerable GetExportableVariableNames(this IGlobalScope scope) - // drop imported modules and typing - => scope.Variables - .Where(v => { - // Instances are always fine. - if (v.Value is IPythonInstance) { - return true; - } - - var valueType = v.Value?.GetPythonType(); - switch (valueType) { - case PythonModule _: - case IPythonFunctionType f when f.IsLambda(): - return false; // Do not re-export modules. - } - - if (scope.Module is TypingModule) { - return true; // Let typing module behave normally. - } - - // Do not re-export types from typing. However, do export variables - // assigned with types from typing. Example: - // from typing import Any # do NOT export Any - // x = Union[int, str] # DO export x - return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; - }) - .Select(v => v.Name) - .ToArray(); - public static IMember LookupNameInScopes(this IScope currentScope, string name, out IScope scope) { scope = null; foreach (var s in currentScope.EnumerateTowardsGlobal) { diff --git a/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs b/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs index 065a8c1c0..54ea7c16e 100644 --- a/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs @@ -13,10 +13,17 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis { public static class VariableExtensions { public static T GetValue(this IVariable v) where T : class => v.Value as T; + + public static bool IsTypeInfo(this IVariable v) => v.Value is IPythonType; + public static bool IsTypeInfoOf(this IVariable v) where T : class, IPythonType => v.Value is T; + + public static bool IsInstance(this IVariable v) => v.Value is IPythonInstance; + public static bool IsInstanceOf(this IVariable v) where T: class, IPythonInstance => v.Value is T; } } diff --git a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj index 4dfae34ee..a244ea8fb 100644 --- a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj +++ b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj @@ -1025,4 +1025,4 @@ - \ No newline at end of file + diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 5edae0a92..c8caf615c 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -40,7 +40,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth private IPythonType _boolType; public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services) - : base(moduleName, ModuleType.Builtins, filePath, null, false, services) { } // TODO: builtins stub & persistence + : base(moduleName, ModuleType.Builtins, filePath, null, false, false, services) { } // TODO: builtins stub & persistence #region IMemberContainer public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name); @@ -67,6 +67,9 @@ protected override void Analyze(PythonAst ast, int version) { protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); + foreach (var n in GetMemberNames()) { + GetMember(n).GetPythonType()?.MakeReadOnly(); + } base.OnAnalysisComplete(); } diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs index 451cdbaea..bf1e3d0c5 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents compiled module that is built into the language. /// internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule { - public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services) - : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, false, services) { } + public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, bool isPersistent, IServiceContainer services) + : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, isPersistent, false, services) { } protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sm) ? null : new [] { "-W", "ignore", "-B", "-E", sm, "-u8", Name }; diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index 345d29754..a5ddf50db 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -26,8 +26,8 @@ namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache; - public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isTypeshed, IServiceContainer services) - : base(moduleName, filePath, moduleType, stub, isTypeshed, services) { } + public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isPersistent, bool isTypeshed, IServiceContainer services) + : base(moduleName, filePath, moduleType, stub, isPersistent, isTypeshed, services) { } public override string Documentation => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty; diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs new file mode 100644 index 000000000..40ea4c0a6 --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleCache.cs @@ -0,0 +1,22 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Modules { + public interface IModuleCache { + string GetCacheFilePath(string filePath); + string ReadCachedModule(string filePath); + void WriteCachedModule(string filePath, string code); + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index a74e221c3..91a034f98 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -14,6 +14,8 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; +using System.Threading; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Types; @@ -86,5 +88,7 @@ public interface IModuleManagement : IModuleResolution { ImmutableArray LibraryPaths { get; } bool SetUserConfiguredPaths(ImmutableArray paths); + + IEnumerable GetImportedModules(CancellationToken cancellationToken); } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs index 493919b30..4b8ce7744 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; @@ -45,11 +44,5 @@ public interface IModuleResolution { /// Reloads all modules. Typically after installation or removal of packages. /// Task ReloadAsync(CancellationToken token = default); - - /// - /// Returns collection of all currently imported modules. - /// - /// - IEnumerable GetImportedModules(CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs index 876e8ba47..ff57426cc 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs @@ -48,6 +48,11 @@ public sealed class ModuleCreationOptions { /// public IPythonModule Stub { get; set; } + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; set; } + /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs new file mode 100644 index 000000000..1e0e4752a --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Modules { + internal sealed class DependencyProvider: IDependencyProvider { + private readonly IPythonModule _module; + private readonly IModuleDatabaseService _dbService; + + public static IDependencyProvider Empty { get; } = new EmptyDependencyProvider(); + + public DependencyProvider(IPythonModule module, IServiceContainer services) { + _dbService = services.GetService(); + _module = module; + } + + #region IDependencyProvider + public ISet GetDependencies(PythonAst ast) { + if (_dbService != null && _dbService.TryRestoreDependencies(_module, out var dp)) { + return dp.GetDependencies(ast); + } + + // TODO: try and handle LoadFunctionDependencyModules functionality here. + var dw = new DependencyWalker(_module, ast); + return dw.Dependencies; + } + #endregion + + private sealed class EmptyDependencyProvider: IDependencyProvider { + public ISet GetDependencies(PythonAst ast) => new HashSet(); + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index dba50e7e7..031cef760 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -27,6 +27,7 @@ using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -76,13 +77,14 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser SetDeclaringModule(this); } - protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isTypeshed, IServiceContainer services) : + protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isPersistent, bool isTypeshed, IServiceContainer services) : this(new ModuleCreationOptions { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType, Stub = stub, - IsTypeshed = isTypeshed + IsTypeshed = isTypeshed, + IsPersistent = isPersistent }, services) { } internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services) @@ -108,6 +110,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s ModuleState = ModuleState.Analyzed; } + IsPersistent = creationOptions.IsPersistent; IsTypeshed = creationOptions.IsTypeshed; InitializeContent(creationOptions.Content, 0); @@ -150,8 +153,35 @@ public virtual string Documentation { #region IMemberContainer public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; - public virtual IEnumerable GetMemberNames() => GlobalScope.GetExportableVariableNames(); + public virtual IEnumerable GetMemberNames() { + // drop imported modules and typing. + return GlobalScope.Variables + .Where(v => { + // Instances are always fine. + if (v.Value is IPythonInstance) { + return true; + } + + var valueType = v.Value?.GetPythonType(); + switch (valueType) { + case PythonModule _: + case IPythonFunctionType f when f.IsLambda(): + return false; // Do not re-export modules. + } + if (this is TypingModule) { + return true; // Let typing module behave normally. + } + + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; + }) + .Select(v => v.Name) + .ToArray(); + } #endregion #region ILocatedMember @@ -183,6 +213,11 @@ public virtual string Documentation { /// public IPythonModule PrimaryModule { get; private set; } + /// + /// Indicates if module is restored from database. + /// + public bool IsPersistent { get; } + /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. @@ -323,7 +358,7 @@ private void Parse(CancellationToken cancellationToken) { int version; Parser parser; - // Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name} ({ModuleType})"); + //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); lock (_syncObj) { version = _buffer.Version; @@ -379,6 +414,8 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #endregion #region IAnalyzable + public virtual IDependencyProvider DependencyProvider => new DependencyProvider(this, Services); + public void NotifyAnalysisBegins() { lock (_syncObj) { if (_updated) { @@ -408,8 +445,7 @@ public void NotifyAnalysisBegins() { } public ModuleWalker Analyze(PythonAst ast) { - var eval = new ExpressionEval(Services, this, ast); - var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance); + var walker = new ModuleWalker(Services, this, ast, CancellationToken.None); ast.Walk(walker); walker.Complete(); return walker; @@ -502,12 +538,17 @@ private void InitializeContent(string content, int version) { Parse(); } } + Services.GetService().InvalidateAnalysis(this); } private void SetOrLoadContent(string content) { if (ModuleState < ModuleState.Loading) { try { - content = content ?? LoadContent(); + if (IsPersistent) { + content = string.Empty; + } else { + content = content ?? LoadContent(); + } _buffer.SetContent(content); ModuleState = ModuleState.Loaded; } catch (IOException) { } catch (UnauthorizedAccessException) { } diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index 94215dd5d..51550546f 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -37,6 +37,7 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public IPythonModule Module { get; } public IPythonInterpreter Interpreter { get; } + public ModuleState ModuleState => Module?.ModuleState ?? ModuleState.None; public IDocumentAnalysis Analysis => Module?.Analysis; public string Documentation => Module?.Documentation ?? string.Empty; @@ -51,9 +52,8 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public BuiltinTypeId TypeId => BuiltinTypeId.Module; public Uri Uri => Module?.Uri; public override PythonMemberType MemberType => PythonMemberType.Module; + public bool IsPersistent => Module?.IsPersistent == true; public bool IsTypeshed => Module?.IsTypeshed == true; - public ModuleState ModuleState => Module?.ModuleState ?? ModuleState.None; - public IEnumerable ChildrenNames => _children.Keys; public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) { Name = name; @@ -70,7 +70,7 @@ public PythonVariableModule(IPythonModule module): base(module) { public void AddChildModule(string memberName, PythonVariableModule module) => _children[memberName] = module; public IMember GetMember(string name) => _children.TryGetValue(name, out var module) ? module : Module?.GetMember(name); - public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(ChildrenNames).Distinct() : ChildrenNames; + public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(_children.Keys).Distinct() : _children.Keys; public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName); public IMember Index(IPythonInstance instance, IArgumentSet args) => Interpreter.UnknownType; @@ -79,11 +79,15 @@ public PythonVariableModule(IPythonModule module): base(module) { public bool Equals(IPythonModule other) => other is PythonVariableModule module && Name.EqualsOrdinal(module.Name); public override bool Equals(object obj) => Equals(obj as IPythonModule); - public override int GetHashCode() => Name.GetHashCode(); + public override int GetHashCode() => 0; #region ILocationConverter public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default; public int LocationToIndex(SourceLocation location) => (Module as ILocationConverter)?.LocationToIndex(location) ?? default; #endregion + + #region IDependencyProvider + public IDependencyProvider DependencyProvider => (Module as IAnalyzable)?.DependencyProvider; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 03913c364..22dafdadd 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -56,7 +56,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr public IBuiltinsPythonModule BuiltinsModule => _builtins; - public IEnumerable GetImportedModules(CancellationToken cancellationToken = default) { + public IEnumerable GetImportedModules(CancellationToken cancellationToken) { foreach (var module in _specialized.Values) { cancellationToken.ThrowIfCancellationRequested(); yield return module; @@ -77,54 +77,40 @@ protected override IPythonModule CreateModule(string name) { return null; } - IPythonModule module; if (moduleImport.ModulePath != null) { - module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); + var module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); if (module != null) { GetRdt().LockDocument(module.Uri); return module; } } - var moduleType = moduleImport.IsBuiltin ? ModuleType.CompiledBuiltin - : moduleImport.IsCompiled ? ModuleType.Compiled - : moduleImport.IsLibrary ? ModuleType.Library - : ModuleType.User; - var dbs = GetDbService(); - if (dbs != null) { - var sw = Stopwatch.StartNew(); - module = dbs.RestoreModule(name, moduleImport.ModulePath, moduleType); - sw.Stop(); - if (module != null) { - Log?.Log(TraceEventType.Verbose, $"Restored from database: {name} in {sw.ElapsedMilliseconds} ms."); - Interpreter.ModuleResolution.SpecializeModule(name, x => module, true); - return module; + moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath); + + IPythonModule stub = null; + if (!moduleImport.IsPersistent) { + // If there is a stub, make sure it is loaded and attached + // First check stub next to the module. + if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { + // If nothing found, try Typeshed. + stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); } - } - // If there is a stub, make sure it is loaded and attached - // First check stub next to the module. - if (TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - Analyzer.InvalidateAnalysis(stub); - } else { - // If nothing found, try Typeshed. - stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); - } - - // If stub is created and its path equals to module, return that stub as module - if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { - return stub; + // If stub is created and its path equals to module, return that stub as module + if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { + return stub; + } } if (moduleImport.IsBuiltin) { Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - return new CompiledBuiltinPythonModule(name, stub, Services); + return new CompiledBuiltinPythonModule(name, stub, moduleImport.IsPersistent, Services); } if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, false, Services); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, false, Services); } Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); @@ -132,9 +118,10 @@ protected override IPythonModule CreateModule(string name) { var mco = new ModuleCreationOptions { ModuleName = moduleImport.FullName, - ModuleType = moduleType, + ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, FilePath = moduleImport.ModulePath, - Stub = stub + Stub = stub, + IsPersistent = moduleImport.IsPersistent }; return GetRdt().AddModule(mco); @@ -196,7 +183,7 @@ private void AddBuiltinTypesToPathResolver() { Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly."); // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); - var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember; + var value = (builtinModuleNamesMember as IVariable)?.Value ?? builtinModuleNamesMember; if (value.TryGetConstant(out var s)) { var builtinModuleNames = s.Split(',').Select(n => n.Trim()); PathResolver.SetBuiltins(builtinModuleNames); diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 52112e7bf..0a3782b24 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -18,7 +18,6 @@ using System.IO; using System.Linq; using System.Threading; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; @@ -27,13 +26,13 @@ using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Modules.Resolution { internal abstract class ModuleResolutionBase { protected IServiceContainer Services { get; } protected IFileSystem FileSystem { get; } protected IPythonInterpreter Interpreter { get; } - protected IPythonAnalyzer Analyzer { get; } protected ILogger Log { get; } protected ConcurrentDictionary Modules { get; } = new ConcurrentDictionary(); @@ -50,7 +49,6 @@ protected ModuleResolutionBase(string root, IServiceContainer services) { FileSystem = services.GetService(); Interpreter = services.GetService(); - Analyzer = services.GetService(); StubCache = services.GetService(); Log = services.GetService(); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 91576a8dc..971fe86c5 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -48,49 +48,23 @@ public TypeshedResolution(string root, IServiceContainer services) : base(root, } protected override IPythonModule CreateModule(string name) { - if (!TryCreateStubModule(name, out var module)) { - return null; - } - - Analyzer.InvalidateAnalysis(module); - return module; - - } - - private bool TryCreateStubModule(string name, out IPythonModule module) { - module = null; var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); if (moduleImport != null) { if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath); - return false; + return null; } - - module = new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, Services); - return true; + return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, Services); } var i = name.IndexOf('.'); if (i == 0) { Debug.Fail("Invalid module name"); - return false; + return null; } var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => FileSystem.FileExists(p)); - if (stubPath != null) { - module = new StubPythonModule(name, stubPath, true, Services); - return true; - } - return false; - } - - public IEnumerable GetImportedModules(CancellationToken cancellationToken = default) { - foreach (var moduleRef in Modules.Values) { - cancellationToken.ThrowIfCancellationRequested(); - if (moduleRef.Value != null) { - yield return moduleRef.Value; - } - } + return stubPath != null ? new StubPythonModule(name, stubPath, true, Services) : null; } public Task ReloadAsync(CancellationToken cancellationToken = default) { diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index 41bd338b2..f9f545348 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; @@ -31,11 +32,15 @@ namespace Microsoft.Python.Analysis.Modules { /// internal abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) - : base(name, modulePath, ModuleType.Specialized, null, false, services) { } + : base(name, modulePath, ModuleType.Specialized, null, false, false, services) { } protected override string LoadContent() { // Exceptions are handled in the base return FileSystem.FileExists(FilePath) ? FileSystem.ReadTextWithRetry(FilePath) : string.Empty; } + + #region IAnalyzable + public override IDependencyProvider DependencyProvider => Modules.DependencyProvider.Empty; + #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs index d01eaadfe..7709c36c0 100644 --- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Modules { /// internal class StubPythonModule : CompiledPythonModule { public StubPythonModule(string moduleName, string stubPath, bool isTypeshed, IServiceContainer services) - : base(moduleName, ModuleType.Stub, stubPath, null, isTypeshed, services) { + : base(moduleName, ModuleType.Stub, stubPath, null, false, isTypeshed, services) { } protected override string LoadContent() { diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs index 00bf93bae..61f35325a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs @@ -21,16 +21,16 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// public interface ITypingNamedTupleType : ITypingTupleType { IReadOnlyList ItemNames { get; } - /// - /// Allows setting alternative name to the tuple at the variable assignment time. - /// - /// - /// Named tuple may get assigned to variables that have name different from the tuple itself. - /// Then the name may conflict with other types in module or its persistent model. For example, - /// 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) but there is also - /// 'class TokenInfo(_TokenInfo)'' so we have to use the variable name in order to avoid type conflicts. - /// - /// + /// + /// Allows setting alternative name to the tuple at the variable assignment time. + /// + /// + /// Named tuple may get assigned to variables that have name different from the tuple itself. + /// Then the name may conflict with other types in module or its persistent model. For example, + /// 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) but there is also + /// 'class TokenInfo(_TokenInfo)'' so we have to use the variable name in order to avoid type conflicts. + /// + /// void SetName(string name); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs index 98be2ad23..f568fd9fe 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs @@ -58,7 +58,7 @@ public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOn #region IPythonType public override string Name => _name; - public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; + public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; // Named tuple name is a type name as class. public override bool IsSpecialized => true; public override string Documentation { get; } #endregion diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs index 7453c869c..9b438e082 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs @@ -64,6 +64,11 @@ public interface IPythonModule : IPythonType { /// IPythonModule PrimaryModule { get; } + /// + /// Indicates if module is restored from database. + /// + bool IsPersistent { get; } + /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs index a6efd3d4f..badcecb6b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs @@ -25,6 +25,11 @@ public interface IPythonPropertyType : IPythonClassMember { /// FunctionDefinition FunctionDefinition { get; } + /// + /// A user readable description of the property. + /// + string Description { get; } + /// /// True if the property is read-only. /// diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 504af4be9..288a9fe35 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -91,18 +91,16 @@ public virtual void RemoveReferences(IPythonModule module) { } internal abstract class EmptyLocatedMember : ILocatedMember { - protected EmptyLocatedMember(IPythonModule declaringModule, PythonMemberType memberType) { - DeclaringModule = declaringModule; + protected EmptyLocatedMember(PythonMemberType memberType) { MemberType = memberType; - Location = new Location(DeclaringModule); } public PythonMemberType MemberType { get; } - public IPythonModule DeclaringModule { get; } + public IPythonModule DeclaringModule => null; public LocationInfo Definition => LocationInfo.Empty; public IReadOnlyList References => Array.Empty(); public void AddReference(Location location) { } public void RemoveReferences(IPythonModule module) { } - public Location Location { get; } + public Location Location { get; internal set; } } } diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index 88a867020..53060d859 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -23,8 +23,6 @@ public Location(IPythonModule module, IndexSpan indexSpan = default) { IndexSpan = indexSpan; } - public void Deconstruct(out IPythonModule module, out IndexSpan indexSpan) => (module, indexSpan) = (Module, IndexSpan); - public IPythonModule Module { get; } public IndexSpan IndexSpan { get; } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index b31a42712..bb58c303a 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -59,7 +59,7 @@ internal partial class PythonClassType { /// B[int] inherits from A[int, str] /// public virtual IPythonType CreateSpecificType(IArgumentSet args) { - lock (MembersLock) { + lock (_membersLock) { var newGenericTypeParameters = GetTypeParameters(); var newBases = new List(); diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index e6f805482..3ba89e2a0 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -37,7 +37,9 @@ internal enum ClassDocumentationSource { Base } private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; + private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); + private readonly object _membersLock = new object(); private List _bases; private IReadOnlyList _mro; @@ -67,7 +69,7 @@ public PythonClassType( public override PythonMemberType MemberType => PythonMemberType.Class; public override IEnumerable GetMemberNames() { - lock (MembersLock) { + lock (_membersLock) { var names = new HashSet(Members.Keys); foreach (var m in Mro.Skip(1)) { names.UnionWith(m.GetMemberNames()); @@ -77,7 +79,7 @@ public override IEnumerable GetMemberNames() { } public override IMember GetMember(string name) { - lock (MembersLock) { + lock (_membersLock) { if (Members.TryGetValue(name, out var member)) { return member; } @@ -185,7 +187,7 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases { get { - lock (MembersLock) { + lock (_membersLock) { return _bases?.ToArray(); } } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index b287b7c1f..e394de6a2 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -17,7 +17,6 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -125,20 +124,10 @@ public string GetReturnDocumentation(IPythonType self = null) { public IMember Call(IArgumentSet args, IPythonType self) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(args.Eval?.Module, this, args, default); + var rt = _returnValueProvider?.Invoke(args.Eval.Module, this, args, default); if (!rt.IsUnknown()) { return rt; } - if (StaticReturnValue == null && !string.IsNullOrEmpty(_returnDocumentation) && FunctionDefinition?.ReturnAnnotation != null) { - // There is return documentation but no static return value. - // This may happen if function is inside module circular - // dependency loop. Try and re-evaluate now. - var returnValue = FunctionEvaluator.GetReturnValueFromAnnotation(args.Eval as ExpressionEval, FunctionDefinition.ReturnAnnotation); - if (returnValue != null) { - SetReturnValue(returnValue, true); - return returnValue; - } - } } return GetSpecificReturnType(self as IPythonClassType, args); diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 33d75e15b..12297dace 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -14,10 +14,13 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { internal sealed class PythonPropertyType : PythonType, IPythonPropertyType { + private IPythonFunctionOverload _getter; + public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) : this(fd.Name, location, fd.GetDocumentation(), declaringType, isAbstract) { declaringType.DeclaringModule.AddAstNode(this, fd); @@ -40,13 +43,19 @@ public PythonPropertyType(string name, Location location, string documentation, public bool IsReadOnly => true; public IPythonType DeclaringType { get; } + public string Description { + get { + var typeName = ReturnType?.GetPythonType()?.Name; + return typeName != null ? Resources.PropertyOfType.FormatUI(typeName) : Resources.PropertyOfUnknownType; + } + } + public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => Getter.Call(args, instance?.GetPythonType() ?? DeclaringType); + => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); - public IMember ReturnType => Getter?.Call(ArgumentSet.WithoutContext, DeclaringType); + public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType); #endregion - internal void AddOverload(PythonFunctionOverload overload) => Getter = overload; - internal PythonFunctionOverload Getter { get; private set; } + internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; } } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index cdfe7fb81..11770248a 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -19,20 +19,22 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{" + nameof(Name) + "}")] internal class PythonType : LocatedMember, IPythonType { + private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; + private bool _readonly; - protected object MembersLock { get; } = new object(); protected IReadOnlyDictionary Members => WritableMembers; private Dictionary WritableMembers => _members ?? (_members = new Dictionary()); - public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) + public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) : this(name, location, typeId) { BaseName = name ?? throw new ArgumentNullException(nameof(name)); Documentation = documentation; @@ -86,19 +88,8 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume #endregion #region IMemberContainer - - public virtual IMember GetMember(string name) { - lock (MembersLock) { - return Members.TryGetValue(name, out var member) ? member : null; - } - } - - public virtual IEnumerable GetMemberNames() { - lock (MembersLock) { - return Members.Keys.ToArray(); - } - } - + public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; + public virtual IEnumerable GetMemberNames() => Members.Keys; #endregion internal bool TrySetTypeId(BuiltinTypeId typeId) { @@ -112,25 +103,29 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { - lock (MembersLock) { - foreach (var v in variables.OfType()) { - var hasMember = Members.ContainsKey(v.Name); - if (overwrite || !hasMember) { - // If variable holds function or a class, use value as member. - // If it holds an instance, use the variable itself (i.e. it is a data member). - WritableMembers[v.Name] = v.Value; - } - if (hasMember) { - v.IsClassMember = true; + lock (_lock) { + if (!_readonly) { + foreach (var v in variables.OfType()) { + var hasMember = Members.ContainsKey(v.Name); + if (overwrite || !hasMember) { + // If variable holds function or a class, use value as member. + // If it holds an instance, use the variable itself (i.e. it is a data member). + WritableMembers[v.Name] = v.Value; + } + if (hasMember) { + v.IsClassMember = true; + } } } } } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (MembersLock) { - foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { - WritableMembers[kv.Key] = kv.Value; + lock (_lock) { + if (!_readonly) { + foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { + WritableMembers[kv.Key] = kv.Value; + } } } } @@ -144,22 +139,24 @@ internal void AddMembers(IPythonClassType cls, bool overwrite) { } internal IMember AddMember(string name, IMember member, bool overwrite) { - lock (MembersLock) { - if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member is IVariable v ? v.Value : member; + lock (_lock) { + if (!_readonly) { + if (overwrite || !Members.ContainsKey(name)) { + WritableMembers[name] = member; + } } return member; } } - internal bool IsHidden => ContainsMember("__hidden__"); - - protected bool ContainsMember(string name) { - lock (MembersLock) { - return Members.ContainsKey(name); + internal void MakeReadOnly() { + lock (_lock) { + _readonly = true; } } + internal bool IsHidden => ContainsMember("__hidden__"); + protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; } } diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index d84b87a3c..5b129ac91 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -60,16 +60,10 @@ protected PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaring _builtinTypeId = builtinTypeId; } - protected PythonTypeWrapper() { } - protected void SetInnerType(IPythonType innerType) { - _innerType = innerType; - DeclaringModule = _innerType.DeclaringModule; - } - #region IPythonType public virtual string Name => _typeName ?? InnerType.Name; public virtual string QualifiedName => _typeName != null ? $"{DeclaringModule.Name}:{_typeName}" : InnerType.QualifiedName; - public IPythonModule DeclaringModule { get; private set; } + public IPythonModule DeclaringModule { get; } public virtual string Documentation => _documentation ?? InnerType.Documentation; public virtual BuiltinTypeId TypeId => InnerType.TypeId; public virtual PythonMemberType MemberType => InnerType.MemberType; diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs index ae38c78f3..53fc8175d 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs @@ -62,7 +62,7 @@ public IReadOnlyList Items _contents.TryGetValue(key, out var value) ? value : UnknownType; public override IPythonIterator GetIterator() => - Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule); + Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); public override IMember Index(IArgumentSet args) { if (args.Arguments.Count == 1) { diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs index 06f895779..2cc207822 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs @@ -46,8 +46,8 @@ public override IMember Call(string memberName, IArgumentSet args) { /// Empty iterator /// internal sealed class EmptyIterator : EmptyLocatedMember, IPythonIterator { - public EmptyIterator(IPythonModule declaringModule): base(declaringModule, PythonMemberType.Class) { - Type = declaringModule.Interpreter.UnknownType; + public EmptyIterator(IPythonType unknownType): base(PythonMemberType.Class) { + Type = unknownType; } public IPythonIterator GetIterator() => this; diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index 7e2bf7695..6b350178d 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -48,9 +48,9 @@ public interface IScope { /// IReadOnlyList Children { get; } - /// - /// Locates child scope by name. - /// + /// + /// Locates child scope by name. + /// IScope GetChildScope(ScopeStatement node); /// diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index ed323da8d..0316402cf 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -15,9 +15,6 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core.Collections; -using Microsoft.Python.Core.Diagnostics; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Values { @@ -44,14 +41,14 @@ private void DeclareBuiltinVariables() { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var objectType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - DeclareBuiltinVariable("__debug__", boolType, location); - DeclareBuiltinVariable("__doc__", strType, location); - DeclareBuiltinVariable("__file__", strType, location); - DeclareBuiltinVariable("__name__", strType, location); - DeclareBuiltinVariable("__package__", strType, location); - DeclareBuiltinVariable("__path__", listType, location); - DeclareBuiltinVariable("__dict__", dictType, location); - DeclareBuiltinVariable("__spec__", objectType, location); + DeclareVariable("__debug__", boolType, VariableSource.Builtin, location); + DeclareVariable("__doc__", strType, VariableSource.Builtin, location); + DeclareVariable("__file__", strType, VariableSource.Builtin, location); + DeclareVariable("__name__", strType, VariableSource.Builtin, location); + DeclareVariable("__package__", strType, VariableSource.Builtin, location); + DeclareVariable("__path__", listType, VariableSource.Builtin, location); + DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); + DeclareVariable("__spec__", objectType, VariableSource.Builtin, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/PythonInstance.cs b/src/Analysis/Ast/Impl/Values/PythonInstance.cs index 43ef4969e..52ae31aee 100644 --- a/src/Analysis/Ast/Impl/Values/PythonInstance.cs +++ b/src/Analysis/Ast/Impl/Values/PythonInstance.cs @@ -61,7 +61,7 @@ public virtual IPythonIterator GetIterator() { return new PythonInstanceIterator(instance, Type.DeclaringModule.Interpreter); } - return new EmptyIterator(Type.DeclaringModule); + return new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); } public bool Equals(IPythonInstance other) => Type?.Equals(other?.Type) == true; diff --git a/src/Analysis/Ast/Impl/Values/PythonNone.cs b/src/Analysis/Ast/Impl/Values/PythonNone.cs index 78112f8f5..3af3360d8 100644 --- a/src/Analysis/Ast/Impl/Values/PythonNone.cs +++ b/src/Analysis/Ast/Impl/Values/PythonNone.cs @@ -26,7 +26,7 @@ internal sealed class PythonNone : PythonType, IPythonInstance { public IMember Call(string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; - public IPythonIterator GetIterator() => new EmptyIterator(DeclaringModule); + public IPythonIterator GetIterator() => new EmptyIterator(DeclaringModule.Interpreter.UnknownType); public IMember Index(IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 08df8a957..7ef4408fd 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -19,7 +19,6 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Values { @@ -34,8 +33,6 @@ internal class Scope : IScope { private Dictionary _childScopes; public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { - Check.ArgumentNotNull(nameof(module), module); - OuterScope = outerScope; Module = module; if (node != null) { @@ -52,7 +49,6 @@ public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { public IReadOnlyList Children => _childScopes?.Values.ToArray() ?? Array.Empty(); public IScope GetChildScope(ScopeStatement node) => _childScopes != null && _childScopes.TryGetValue(node, out var s) ? s : null; - public IVariableCollection Variables => _variables ?? VariableCollection.Empty; public IVariableCollection NonLocals => _nonLocals ?? VariableCollection.Empty; public IVariableCollection Globals => _globals ?? VariableCollection.Empty; @@ -96,7 +92,8 @@ public void DeclareImported(string name, IMember value, Location location = defa => (_imported ?? (_imported = new VariableCollection())).DeclareVariable(name, value, VariableSource.Import, location); #endregion - internal void AddChildScope(Scope s) => (_childScopes ?? (_childScopes = new Dictionary()))[s.Node] = s; + internal void AddChildScope(Scope s) + => (_childScopes ?? (_childScopes = new Dictionary()))[s.Node] = s; internal void ReplaceVariable(IVariable v) { VariableCollection.RemoveVariable(v.Name); @@ -114,7 +111,7 @@ private void DeclareBuiltinVariables() { var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - DeclareBuiltinVariable("__name__", strType, location); + VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, location); if (Node is FunctionDefinition) { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); @@ -131,8 +128,7 @@ private void DeclareBuiltinVariables() { DeclareBuiltinVariable("__self__", objType, location); } } - - protected void DeclareBuiltinVariable(string name, IPythonType type, Location location) + protected void DeclareBuiltinVariable(string name, IPythonType type, Location location) => VariableCollection.DeclareVariable(name, type, VariableSource.Builtin, location); } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 5ad5426de..db915ae7f 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -37,26 +37,18 @@ using TestUtilities; namespace Microsoft.Python.Analysis.Tests { - public abstract class AnalysisTestBase: IDisposable { - private readonly CancellationTokenSource _testCts; + public abstract class AnalysisTestBase { + protected const int AnalysisTimeoutInMS = 1000 * 60; - protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(3000 * 60); + protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(AnalysisTimeoutInMS); + + private TimeSpan GetAnalysisTimeout() => Debugger.IsAttached ? Timeout.InfiniteTimeSpan : AnalysisTimeout; protected TestLogger TestLogger { get; } = new TestLogger(); protected ServiceManager Services { get; private set; } protected virtual IDiagnosticsService GetDiagnosticsService(IServiceContainer s) => null; - protected CancellationToken TestCancellationToken => _testCts.Token; - - protected AnalysisTestBase() { - _testCts = new CancellationTokenSource(Debugger.IsAttached ? Timeout.InfiniteTimeSpan : AnalysisTimeout); - } - - public void Dispose() { - _testCts.Dispose(); - } - protected ServiceManager CreateServiceManager() { Services = new ServiceManager(); @@ -167,10 +159,14 @@ protected async Task GetAnalysisAsync( var ast = await doc.GetAstAsync(CancellationToken.None); ast.Should().NotBeNull(); TestLogger.Log(TraceEventType.Information, "Test: AST end."); + TestLogger.Log(TraceEventType.Information, "Test: Analysis begin."); - await services.GetService().WaitForCompleteAnalysisAsync(TestCancellationToken); - var analysis = await doc.GetAnalysisAsync(-1, TestCancellationToken); + IDocumentAnalysis analysis; + using (var cts = new CancellationTokenSource(GetAnalysisTimeout())) { + await services.GetService().WaitForCompleteAnalysisAsync(cts.Token); + analysis = await doc.GetAnalysisAsync(-1, cts.Token); + } analysis.Should().NotBeNull(); TestLogger.Log(TraceEventType.Information, "Test: Analysis end."); @@ -180,8 +176,10 @@ protected async Task GetAnalysisAsync( protected async Task GetDocumentAnalysisAsync(IDocument document) { var analyzer = Services.GetService(); - await analyzer.WaitForCompleteAnalysisAsync(TestCancellationToken); - return await document.GetAnalysisAsync(Timeout.Infinite, TestCancellationToken); + using (var cts = new CancellationTokenSource(GetAnalysisTimeout())) { + await analyzer.WaitForCompleteAnalysisAsync(cts.Token); + return await document.GetAnalysisAsync(Timeout.Infinite, cts.Token); + } } } } diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index 7a5693747..92effbbca 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Dependencies; -using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Core.Collections; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -38,17 +36,16 @@ public void TestInitialize() public void Cleanup() => TestEnvironmentImpl.TestCleanup(); // ReSharper disable StringLiteralTypo - // Square brackets mean that nodes can be walked in parallel. Parentheses mean that nodes are in loop. [DataRow("A:BC|B:C|C", "CBA", "A")] [DataRow("C|A:BC|B:C", "CBA", "A")] - [DataRow("C|B:AC|A:BC", "C(BA)", "A")] - [DataRow("A:CE|B:A|C:B|D:B|E", "E(ACB)D", "D")] - [DataRow("A:D|B:DA|C:BA|D:AE|E", "E(AD)BC", "C")] - [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "(ACB)(DFE)", "F")] - [DataRow("A:BC|B:AC|C:BA|D:BC", "(ABC)D", "D")] + [DataRow("C|B:AC|A:BC", "CBABA", "A")] + [DataRow("A:CE|B:A|C:B|D:B|E", "[CE]ABCABD", "D")] + [DataRow("A:D|B:DA|C:BA|D:AE|E", "[AE]DADBC", "C")] + [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "ABCABCDEFDEF", "F")] + [DataRow("A:BC|B:AC|C:BA|D:BC", "ACBACBD", "D")] [DataRow("A|B|C|D:AB|E:BC", "[ABC][DE]", "D|E")] - [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "E(ACB)[FD]", "D|F")] - [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "(DEF)[ABC]", "A|B|C")] + [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "D|F")] + [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A|B|C")] // ReSharper restore StringLiteralTypo [DataTestMethod] public void ChangeValue(string input, string output, string root) { @@ -64,7 +61,7 @@ public void ChangeValue(string input, string output, string root) { var walker = resolver.CreateWalker(); var result = new StringBuilder(); - var tasks = new List>(); + var tasks = new List>>(); while (walker.Remaining > 0) { var nodeTask = walker.GetNextAsync(default); if (!nodeTask.IsCompleted) { @@ -73,7 +70,7 @@ public void ChangeValue(string input, string output, string root) { } foreach (var task in tasks) { - AppendFirstChar(result, task.Result); + result.Append(task.Result.Value[0]); task.Result.MarkWalked(); task.Result.MoveNext(); } @@ -89,7 +86,7 @@ public void ChangeValue(string input, string output, string root) { result.ToString().Should().Be(output); } - + [TestMethod] public async Task ChangeValue_ChangeToIdentical() { var resolver = new DependencyResolver(); @@ -101,8 +98,8 @@ public async Task ChangeValue_ChangeToIdentical() { var result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); - node.Should().HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); } @@ -115,8 +112,8 @@ public async Task ChangeValue_ChangeToIdentical() { result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); - node.Should().HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); } @@ -136,8 +133,8 @@ public async Task ChangeValue_TwoChanges() { var result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); - node.Should().HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); } @@ -151,8 +148,8 @@ public async Task ChangeValue_TwoChanges() { result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); - node.Should().HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); } @@ -167,30 +164,31 @@ public async Task ChangeValue_MissingKeys() { resolver.ChangeValue("B", "B", false); resolver.ChangeValue("C", "C:D", true, "D"); var walker = resolver.CreateWalker(); - + + var result = new StringBuilder(); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("B") - .And.HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C:D") - .And.HaveOnlyWalkedDependencies(); + result.Append(node.Value[0]); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); walker.MissingKeys.Should().Equal("D"); + result.ToString().Should().Be("BC"); resolver.ChangeValue("D", "D", false); walker = resolver.CreateWalker(); - walker.MissingKeys.Should().BeEmpty(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:B"); + result = new StringBuilder(); + result.Append((await walker.GetNextAsync(default)).Value[0]); + result.Append((await walker.GetNextAsync(default)).Value[0]); - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("D"); + walker.MissingKeys.Should().BeEmpty(); + result.ToString().Should().Be("AD"); } [TestMethod] @@ -202,9 +200,9 @@ public async Task ChangeValue_Add() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("B", "D"); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BD") - .And.HaveMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeTrue(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -214,46 +212,46 @@ public async Task ChangeValue_Add() { resolver.ChangeValue("B", "B", false); walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); - + node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("B") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("B"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BD") - .And.HaveMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeTrue(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); walker.Remaining.Should().Be(0); - + // Add D resolver.ChangeValue("D", "D:C", false, "C"); walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("D:C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node = await walker.GetNextAsync(default); + node.Value.Should().Be("D:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BD") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -261,7 +259,7 @@ public async Task ChangeValue_Add() { } [TestMethod] - public async Task ChangeValue_Add_ParallelWalkers() { + public async Task ChangeValue_Add_ParallelWalkers() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:BD", true, "B", "D"); resolver.ChangeValue("B", "B:C", false, "C"); @@ -271,9 +269,9 @@ public async Task ChangeValue_Add_ParallelWalkers() { walker.MissingKeys.Should().Equal("D"); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C") - .And.HaveNoMissingDependencies() - .And.HaveValidVersion(); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.IsValidVersion.Should().BeTrue(); // Add D resolver.ChangeValue("D", "D:C", false, "C"); @@ -281,24 +279,24 @@ public async Task ChangeValue_Add_ParallelWalkers() { newWalker.MissingKeys.Should().BeEmpty(); // MarkWalked node from old walker - node.Should().HaveOnlyWalkedDependencies() - .And.HaveInvalidVersion(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("B:C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveInvalidVersion(); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BD") - .And.HaveMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveInvalidVersion(); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeTrue(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); @@ -306,34 +304,34 @@ public async Task ChangeValue_Add_ParallelWalkers() { // Walk new walker node = await newWalker.GetNextAsync(default); - node.Should().HaveSingleValue("C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveValidVersion(); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Should().HaveSingleValue("B:C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveValidVersion(); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Should().HaveSingleValue("D:C") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveValidVersion(); + node.Value.Should().Be("D:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BD") - .And.HaveNoMissingDependencies() - .And.HaveOnlyWalkedDependencies() - .And.HaveValidVersion(); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -353,34 +351,87 @@ public async Task ChangeValue_PartiallyWalkLoop() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("E") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("E"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:CE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("D:BE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveLoopValues("B:CE", "C:DE", "D:BE") - .And.HaveNonWalkedDependencies(); + node.Value.Should().Be("C:DE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); // Create new walker var newWalker = resolver.CreateWalker(); - // Mark vertex walked as it would've been in parallel - // Loops are always walked fully. + // Mark vertex walked as it would've in parallel node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:B") - .And.HaveOnlyWalkedDependencies() - .And.NotBeWalkedWithDependencies(); + node.Value.Should().Be("B:CE"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsWalkedWithDependencies.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); // Now iterate with new walker node = await newWalker.GetNextAsync(default); - node.Should().HaveSingleValue("A:B") - .And.HaveOnlyWalkedDependencies() - .And.BeWalkedWithDependencies(); + node.Value.Should().Be("B:CE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.IsWalkedWithDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("D:BE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.IsWalkedWithDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("C:DE"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.IsWalkedWithDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("B:CE"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsWalkedWithDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("D:BE"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsWalkedWithDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("C:DE"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsWalkedWithDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsWalkedWithDependencies.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); @@ -397,20 +448,20 @@ public async Task ChangeValue_Remove() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("B:C") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BC") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A:BC"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -419,8 +470,8 @@ public async Task ChangeValue_Remove() { walker.MissingKeys.Should().Equal("B"); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BC") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A:BC"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -437,7 +488,7 @@ public async Task ChangeValue_ChangeChangeRemove() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); walker.AffectedValues.Should().Equal("A:B", "B:C", "C:AD"); - walker.Remaining.Should().Be(3); + walker.Remaining.Should().Be(6); //resolver.ChangeValue("D", "D:B", true, "B"); resolver.ChangeValue("A", "A", true); @@ -449,14 +500,14 @@ public async Task ChangeValue_ChangeChangeRemove() { walker.Remaining.Should().Be(2); var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C:AD") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("C:AD"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -474,11 +525,41 @@ public async Task ChangeValue_RemoveFromLoop() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Should().HaveLoopValues("A:B", "B:C", "C:A") - .And.HaveNonWalkedDependencies(); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); node.MarkWalked(); node.MoveNext(); - + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.Remaining.Should().Be(0); resolver.Remove("B"); @@ -486,8 +567,61 @@ public async Task ChangeValue_RemoveFromLoop() { walker.MissingKeys.Should().Equal("B"); node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:B") - .And.HaveOnlyWalkedDependencies(); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_RemoveKeys() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:BC", true, "B", "C"); + resolver.ChangeValue("B", "B:C", false, "C"); + resolver.ChangeValue("C", "C:D", false, "D"); + resolver.ChangeValue("D", "D", false); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().BeEmpty(); + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("D"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:D"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:BC"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + resolver.RemoveKeys("B", "D"); + walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("B", "D"); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:D"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:BC"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); node.MarkWalked(); node.MoveNext(); @@ -505,33 +639,22 @@ public async Task ChangeValue_Skip() { var walker = resolver.CreateWalker(); var result = new StringBuilder(); var node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); + result.Append(node.Value[0]); node.MoveNext(); - + node = await walker.GetNextAsync(default); - AppendFirstChar(result, node); + result.Append(node.Value[0]); node.MoveNext(); - + result.ToString().Should().Be("BD"); resolver.ChangeValue("D", "D", false); walker = resolver.CreateWalker(); result = new StringBuilder(); - AppendFirstChar(result, await walker.GetNextAsync(default)); - AppendFirstChar(result, await walker.GetNextAsync(default)); + result.Append((await walker.GetNextAsync(default)).Value[0]); + result.Append((await walker.GetNextAsync(default)).Value[0]); result.ToString().Should().Be("BD"); } - - private static StringBuilder AppendFirstChar(StringBuilder sb, IDependencyChainNode node) { - switch (node) { - case IDependencyChainSingleNode single: - return sb.Append(single.Value[0]); - case IDependencyChainLoopNode loop: - return sb.Append($"({new string(loop.Values.Select(v => v[0]).ToArray())})"); - default: - throw new InvalidOperationException(); - } - } } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs index 70de139c9..9e4d86965 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Diagnostics.CodeAnalysis; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Text; @@ -22,7 +21,7 @@ namespace Microsoft.Python.Analysis.Tests.FluentAssertions { [ExcludeFromCodeCoverage] internal static class AssertionsFactory { - public static DependencyChainNodeAssertions Should(this IDependencyChainNode node) => new DependencyChainNodeAssertions(node); + public static ScopeAssertions Should(this IScope scope) => new ScopeAssertions(scope); public static MemberAssertions Should(this IMember member) => new MemberAssertions(member); public static PythonFunctionAssertions Should(this IPythonFunctionType f) => new PythonFunctionAssertions(f); @@ -34,7 +33,6 @@ internal static class AssertionsFactory { public static RangeAssertions Should(this Range? range) => new RangeAssertions(range); - public static ScopeAssertions Should(this IScope scope) => new ScopeAssertions(scope); public static SourceSpanAssertions Should(this SourceSpan span) => new SourceSpanAssertions(span); public static SourceSpanAssertions Should(this SourceSpan? span) => new SourceSpanAssertions(span.Value); } diff --git a/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs deleted file mode 100644 index d628ac178..000000000 --- a/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Linq; -using FluentAssertions; -using FluentAssertions.Execution; -using FluentAssertions.Primitives; -using Microsoft.Python.Analysis.Dependencies; -using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; - -namespace Microsoft.Python.Analysis.Tests.FluentAssertions { - internal sealed class DependencyChainNodeAssertions : ReferenceTypeAssertions { - public DependencyChainNodeAssertions(IDependencyChainNode node) { - Subject = node; - } - - protected override string Identifier => nameof(IDependencyChainNode); - - [CustomAssertion] - public AndConstraint HaveSingleValue(T value, string because = "", params object[] reasonArgs) { - var currentStateMessage = Subject == null ? "null" : "loop node"; - - Execute.Assertion.ForCondition(Subject is IDependencyChainSingleNode) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected [{typeof(T)}] node to be single node{{reason}}, but it is {currentStateMessage}"); - - var actual = ((IDependencyChainSingleNode)Subject).Value; - Execute.Assertion.ForCondition(Equals(actual, value)) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected [{typeof(T)}] node to have value {value}{{reason}}, but it has {actual}"); - - return new AndConstraint(this); - } - - [CustomAssertion] - public AndConstraint HaveLoopValues(params T[] values) => HaveLoopValues(values, string.Empty); - - [CustomAssertion] - public AndConstraint HaveLoopValues(T[] values, string because = "", params object[] reasonArgs) { - var currentStateMessage = Subject == null ? "null" : "loop node"; - - Execute.Assertion.ForCondition(Subject is IDependencyChainLoopNode) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected [{typeof(T)}] node to be loop node{{reason}}, but it is {currentStateMessage}"); - - var actual = ((IDependencyChainLoopNode)Subject).Values.ToArray(); - var errorMessage = GetAssertCollectionOnlyContainsMessage(actual, values, "loop node", "value", "values"); - - Execute.Assertion.ForCondition(errorMessage == null) - .BecauseOf(string.Empty, string.Empty) - .FailWith(errorMessage); - - return new AndConstraint(this); - } - - [CustomAssertion] - public AndConstraint HaveOnlyWalkedDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(Subject.HasOnlyWalkedDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have only walked dependencies{{reason}}"); - - return new AndConstraint(this); - } - - [CustomAssertion] - public AndConstraint HaveNonWalkedDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(!Subject.HasOnlyWalkedDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have non-walked dependencies{{reason}}"); - - return new AndConstraint(this); - } - - [CustomAssertion] - public AndConstraint BeWalkedWithDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(Subject.IsWalkedWithDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to be walked with dependencies{{reason}}"); - - return new AndConstraint(this); - } - - [CustomAssertion] - public AndConstraint NotBeWalkedWithDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(!Subject.IsWalkedWithDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to not be walked with dependencies{{reason}}"); - - return new AndConstraint(this); - } - - public AndConstraint HaveMissingDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(Subject.HasMissingDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have missing dependencies{{reason}}"); - - return new AndConstraint(this); - } - - public AndConstraint HaveNoMissingDependencies(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(!Subject.HasMissingDependencies) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have no missing dependencies{{reason}}"); - - return new AndConstraint(this); - } - - public AndConstraint HaveValidVersion(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(Subject.IsValidVersion) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have valid version{{reason}}"); - - return new AndConstraint(this); - } - - public AndConstraint HaveInvalidVersion(string because = "", params object[] reasonArgs) { - Execute.Assertion.ForCondition(!Subject.IsValidVersion) - .BecauseOf(because, reasonArgs) - .FailWith($"Expected {Subject} to have invalid version{{reason}}"); - - return new AndConstraint(this); - } - } -} diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index a177b0626..1f6b10c08 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -21,12 +21,9 @@ using FluentAssertions; using FluentAssertions.Execution; using FluentAssertions.Primitives; -using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; namespace Microsoft.Python.Analysis.Tests.FluentAssertions { @@ -124,125 +121,89 @@ public AndWhichConstraint HaveMember(string return new AndWhichConstraint(this, typedMember); } - public AndConstraint HaveSameMemberNamesAs(IMember member, bool recursive = false) { + public AndConstraint HaveSameMemberNamesAs(IMember member) { member.Should().BeAssignableTo(); return HaveMembers(((IMemberContainer)member).GetMemberNames(), string.Empty); } - private static readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); + public void HaveSameMembersAs(IMember other) { + other.Should().BeAssignableTo(); + var otherContainer = (IMemberContainer)other; - public void HaveSameMembersAs(IMember expected, bool recursive = false, string because = "", params object[] becauseArgs) { - var expectedContainer = expected.Should().BeAssignableTo().Which; - var actualContainer = Subject.GetPythonType(); + var subjectType = Subject.GetPythonType(); + var subjectMemberNames = subjectType.GetMemberNames().ToArray(); + var otherMemberNames = otherContainer.GetMemberNames().ToArray(); - using (_memberGuard.Push(actualContainer, out var reentered)) { - if (reentered) { - return; - } - var actualNames = actualContainer.GetMemberNames().ToArray(); - var expectedNames = expectedContainer.GetMemberNames().Except(Enumerable.Repeat("", 1)).ToArray(); + var missingNames = otherMemberNames.Except(subjectMemberNames).ToArray(); + var extraNames = subjectMemberNames.Except(otherMemberNames).ToArray(); + + Debug.Assert(missingNames.Length == 0); + missingNames.Should().BeEmpty("Subject has missing names: ", missingNames); + + Debug.Assert(extraNames.Length == 0); + extraNames.Should().BeEmpty("Subject has extra names: ", extraNames); - var errorMessage = GetAssertCollectionOnlyContainsMessage(actualNames, expectedNames, GetQuotedName(Subject), "member", "members"); - var assertion = Execute.Assertion.BecauseOf(because, becauseArgs); + foreach (var n in subjectMemberNames.Except(Enumerable.Repeat("__base__", 1))) { + var subjectMember = subjectType.GetMember(n); + var otherMember = otherContainer.GetMember(n); + var subjectMemberType = subjectMember.GetPythonType(); + var otherMemberType = otherMember.GetPythonType(); - Debug.Assert(errorMessage == null); - assertion.ForCondition(errorMessage == null).FailWith(errorMessage); + // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. + if (subjectMember is IPythonInstance) { + otherMember.Should().BeAssignableTo(); + } - // TODO: In restored case __iter__ is a function while in analysis it is a specialized class. - foreach (var n in actualNames.Except(new[] { "__base__", "__iter__" })) { - var actualMember = actualContainer.GetMember(n); - var expectedMember = expectedContainer.GetMember(n); + subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType, $"Type name: {subjectMemberType.Name}"); + //Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); - var actualMemberType = actualMember.GetPythonType(); - var expectedMemberType = expectedMember.GetPythonType(); + if (subjectMemberType is IPythonClassType subjectClass) { + var otherClass = otherMemberType as IPythonClassType; + otherClass.Should().NotBeNull(); - // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. - if (expectedMember is IPythonInstance && !expectedMember.IsUnknown()) { - // Debug.Assert(actualMember is IPythonInstance); - assertion.ForCondition(actualMember is IPythonInstance) - .FailWith($"Expected '{GetName(actualContainer)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); + if (subjectClass is IGenericType gt) { + otherClass.Should().BeAssignableTo(); + otherClass.IsGeneric.Should().Be(gt.IsGeneric, $"Class name: {subjectClass.Name}"); } - Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); - actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); + // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. + //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); + //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); + } - #region Class comparison - if (actualMemberType is IPythonClassType actualClass) { - var expectedClass = expectedMemberType as IPythonClassType; - expectedClass.Should().NotBeNull(); + if (string.IsNullOrEmpty(subjectMemberType.Documentation)) { + otherMemberType.Documentation.Should().BeNullOrEmpty($"Type name: {subjectMemberType.Name}."); + } else { + Debug.Assert(subjectMemberType.Documentation == otherMemberType.Documentation); + subjectMemberType.Documentation.Should().Be(otherMemberType.Documentation, $"Type name: {subjectMemberType.Name}."); + } - if (actualClass is IGenericType gt) { - expectedClass.Should().BeAssignableTo(); - // Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); - // https://github.com/microsoft/python-language-server/issues/1753 - // expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic"); + switch (subjectMemberType.MemberType) { + case PythonMemberType.Class: + // Restored collections (like instance of tuple) turn into classes + // rather than into collections with content since we don't track + // collection content in libraries. We don't compare qualified names + // since original module may be source or a stub and that is not + // preserved during restore. + // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); + break; + case PythonMemberType.Function: + case PythonMemberType.Method: + subjectMemberType.Should().BeAssignableTo(); + otherMemberType.Should().BeAssignableTo(); + if (subjectMemberType is IPythonFunctionType subjectFunction) { + var otherFunction = (IPythonFunctionType)otherMemberType; + subjectFunction.Should().HaveSameOverloadsAs(otherFunction); } - // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. - //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); - //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); - } - #endregion - - #region Documentation comparison - // Allow documentation replacement from primary - // https://github.com/microsoft/python-language-server/issues/1753 - if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { - var expectedDoc = expectedMemberType.Documentation?.Trim(); - var actualDoc = actualMemberType.Documentation?.Trim(); - - Debug.Assert(expectedDoc == actualDoc); - if (string.IsNullOrEmpty(expectedDoc)) { - assertion.ForCondition(string.IsNullOrEmpty(actualDoc)) - .FailWith($"Expected python type of '{GetName(actualMemberType)}.{n}' to have no documentation{{reason}}, but it has '{actualDoc}'"); - } else { - assertion.ForCondition(actualDoc.EqualsOrdinal(expectedDoc)) - .FailWith($"Expected python type of '{GetName(actualMemberType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualDoc}'"); - } - } - #endregion - - #region Member type specific checks - switch (actualMemberType.MemberType) { - case PythonMemberType.Class: - // Restored collections (like instance of tuple) turn into classes - // rather than into collections with content since we don't track - // collection content in libraries. We don't compare qualified names - // since original module may be source or a stub and that is not - // preserved during restore. - // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); - break; - case PythonMemberType.Function: - case PythonMemberType.Method: - actualMemberType.Should().BeAssignableTo(); - expectedMemberType.Should().BeAssignableTo(); - if (actualMemberType is IPythonFunctionType subjectFunction) { - var otherFunction = (IPythonFunctionType)expectedMemberType; - subjectFunction.Should().HaveSameOverloadsAs(otherFunction); - } - - break; - case PythonMemberType.Property: - actualMemberType.Should().BeAssignableTo(); - expectedMemberType.Should().BeAssignableTo(); - break; - case PythonMemberType.Unknown: - actualMemberType.IsUnknown().Should().BeTrue(); - break; - } - #endregion - - // Recurse into members. - // https://github.com/microsoft/python-language-server/issues/1533 - // Ex 'BigEndianStructure' in ctypes has attached members from stub. - // However, when running test fetching it from 'ctypes._endian' yields - // class without stub members. This affects tests with partial restoration. - - // Also, there are issues when restored object methods are - // not specialized like __iter__ or __getattribute__. - if (recursive) { - actualMemberType.Should().HaveSameMembersAs(expectedMemberType, recursive); - } + break; + case PythonMemberType.Property: + subjectMemberType.Should().BeAssignableTo(); + otherMemberType.Should().BeAssignableTo(); + break; + case PythonMemberType.Unknown: + subjectMemberType.IsUnknown().Should().BeTrue(); + break; } } } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 4bc826d31..42f20587b 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -335,8 +336,7 @@ public async Task DeepSubmoduleImport() { await CreateServicesAsync(PythonVersions.LatestAvailable3X); var rdt = Services.GetService(); - var appDoc = rdt.OpenDocument(appUri, @"import top.sub1.sub2.sub3.sub4 -x = top.sub1.sub2.sub3.sub4.f"); + var appDoc = rdt.OpenDocument(appUri, "import top.sub1.sub2.sub3.sub4"); await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); diff --git a/src/Analysis/Ast/Test/LibraryTests.cs b/src/Analysis/Ast/Test/LibraryTests.cs index c34163c03..721fd79d3 100644 --- a/src/Analysis/Ast/Test/LibraryTests.cs +++ b/src/Analysis/Ast/Test/LibraryTests.cs @@ -80,8 +80,8 @@ import requests Assert.Inconclusive("'requests' package is not installed."); } - var r = analysis.Should().HaveVariable("x").OfType("Response") - .Which.Should().HaveMember("encoding").Which.Should().HaveType(BuiltinTypeId.Str); + var r = analysis.Should().HaveVariable("x").OfType("Response").Which; + r.Should().HaveMember("encoding").Which.Should().HaveType(BuiltinTypeId.Str); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Ast/Test/LocationLoopResolverTests.cs b/src/Analysis/Ast/Test/LocationLoopResolverTests.cs deleted file mode 100644 index acf2a50db..000000000 --- a/src/Analysis/Ast/Test/LocationLoopResolverTests.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Linq; -using FluentAssertions; -using Microsoft.Python.Analysis.Dependencies; -using Microsoft.Python.UnitTests.Core.MSTest; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using TestUtilities; - -namespace Microsoft.Python.Analysis.Tests { - [TestClass] - public class LocationLoopResolverTests { - public TestContext TestContext { get; set; } - - [TestInitialize] - public void TestInitialize() - => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - - [TestCleanup] - public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - - // ReSharper disable StringLiteralTypo - [PermutationDataRow("A1B1", "B0C1", "C0A0")] - [PermutationDataRow("A2B8", "B2A0", "B6C0", "C3B4")] - // ReSharper restore StringLiteralTypo - [DataTestMethod] - public void FindStartingItem(params string[] input) { - var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); - LocationLoopResolver.FindStartingItems(edges).Should().Equal('A'); - } - - // ReSharper disable StringLiteralTypo - [PermutationDataRow("A0B1", "B0A1")] - [PermutationDataRow("A0B1", "B0C1", "C0A1")] - // ReSharper restore StringLiteralTypo - [DataTestMethod] - public void NoStartingItem(params string[] input) { - var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); - LocationLoopResolver.FindStartingItems(edges).Should().BeEmpty(); - } - - // ReSharper disable StringLiteralTypo - [PermutationDataRow("A2B4", "B2A0", "C3B4")] - [PermutationDataRow("A2B4", "B2A0", "C2D4", "D2C0")] - // ReSharper restore StringLiteralTypo - [DataTestMethod] - public void TwoStartingItems(params string[] input) { - var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); - LocationLoopResolver.FindStartingItems(edges).Should().BeEquivalentTo('A', 'C'); - } - } -} diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index ba4843fe0..db04110e3 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -47,4 +47,4 @@ - \ No newline at end of file + diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index e6163e5e4..d3278475d 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -94,8 +94,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura var modules = new List(); - foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false) - .Select(f => f.FullName).Where(ModulePath.IsPythonFile)) { + foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false).Select(f => f.FullName).Where(ModulePath.IsPythonFile)) { var mp = ModulePath.FromFullPath(pyd); if (mp.IsDebug) { continue; diff --git a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs index b229ab80e..9d95188e3 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs @@ -27,6 +27,7 @@ public class ModuleImport : IImportChildrenSource { public bool IsCompiled { get; } public bool IsLibrary { get; } public bool IsBuiltin => IsCompiled && ModulePath == null; + public bool IsPersistent { get; set; } public ModuleImport(IImportChildrenSource childrenSource, string name, string fullName, string rootPath, string modulePath, long moduleFileSize, bool isCompiled, bool isLibrary) { _childrenSource = childrenSource; diff --git a/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs b/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs new file mode 100644 index 000000000..b200daf5c --- /dev/null +++ b/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs @@ -0,0 +1,41 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Core.Collections; + +namespace Microsoft.Python.Analysis.Caching { + internal static class DependencyCollectorExtensions { + public static void AddImports(this DependencyCollector dc, IEnumerable imports) { + foreach (var imp in imports) { + foreach (var dottedName in imp.ModuleNames) { + var importNames = ImmutableArray.Empty; + foreach (var part in dottedName.NameParts) { + importNames = importNames.Add(part); + dc.AddImport(importNames, imp.ForceAbsolute); + } + } + } + } + + public static void AddFromImports(this DependencyCollector dc, IEnumerable imports) { + foreach (var imp in imports) { + dc.AddFromImport(imp.RootNames, imp.MemberNames, imp.DotCount, imp.ForceAbsolute); + } + } + } +} diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs index cd258fec9..323d5646f 100644 --- a/src/Caching/Impl/Models/CallableModel.cs +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -32,10 +32,9 @@ internal abstract class CallableModel : MemberModel { [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); - protected CallableModel() { } // For de-serializer from JSON - protected CallableModel(IPythonType callable, IServiceContainer services) { + protected CallableModel(IPythonType callable) { var functions = new List(); var classes = new List(); @@ -51,10 +50,10 @@ protected CallableModel(IPythonType callable, IServiceContainer services) { case IPythonFunctionType ft1 when ft1.IsLambda(): break; case IPythonFunctionType ft2: - functions.Add(new FunctionModel(ft2, services)); + functions.Add(new FunctionModel(ft2)); break; case IPythonClassType cls: - classes.Add(new ClassModel(cls, services)); + classes.Add(new ClassModel(cls)); break; } } @@ -62,7 +61,6 @@ protected CallableModel(IPythonType callable, IServiceContainer services) { Id = callable.Name.GetStableHash(); Name = callable.Name; - DeclaringModuleId = callable.DeclaringModule.GetUniqueId(services); QualifiedName = callable.QualifiedName; Documentation = callable.Documentation; Classes = classes.ToArray(); @@ -82,6 +80,7 @@ protected CallableModel(IPythonType callable, IServiceContainer services) { } } } + protected override IEnumerable GetMemberModels() => Classes.Concat(Functions); } } diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 86e76b530..527816157 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -22,6 +22,7 @@ using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -48,13 +49,11 @@ internal sealed class ClassModel : MemberModel { public GenericParameterValueModel[] GenericParameterValues { get; set; } [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + [NonSerialized] private PythonClassType _cls; public ClassModel() { } // For de-serializer from JSON - /// - /// Constructs class model for persistence off the class in-memory type. - /// - public ClassModel(IPythonClassType cls, IServiceContainer services) { + public ClassModel(IPythonClassType cls) { var methods = new List(); var properties = new List(); var fields = new List(); @@ -79,21 +78,21 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { if (!ct.DeclaringModule.Equals(cls.DeclaringModule)) { continue; } - innerClasses.Add(new ClassModel(ct, services)); + innerClasses.Add(new ClassModel(ct)); break; case IPythonFunctionType ft when ft.IsLambda(): break; case IPythonFunctionType ft when ft.Name == name: - methods.Add(new FunctionModel(ft, services)); + methods.Add(new FunctionModel(ft)); break; case IPythonPropertyType prop when prop.Name == name: - properties.Add(new PropertyModel(prop, services)); + properties.Add(new PropertyModel(prop)); break; case IPythonInstance inst: - fields.Add(VariableModel.FromInstance(name, inst, services)); + fields.Add(VariableModel.FromInstance(name, inst)); break; case IPythonType t: - fields.Add(VariableModel.FromType(name, t, services)); + fields.Add(VariableModel.FromType(name, t)); break; } } @@ -101,15 +100,16 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { Name = cls.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : cls.Name; Id = Name.GetStableHash(); - DeclaringModuleId = cls.DeclaringModule.GetUniqueId(services); QualifiedName = cls.QualifiedName; IndexSpan = cls.Location.IndexSpan.ToModel(); - Documentation = cls.Documentation; - var ntBases = cls.Bases.MaybeEnumerate().OfType().ToArray(); - NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b, services)).ToArray(); + // Only persist documentation from this class, leave bases or __init__ alone. + Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; + + var ntBases = cls.Bases.OfType().ToArray(); + NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray(); - Bases = cls.Bases.MaybeEnumerate().Except(ntBases).Select(t => t.GetPersistentQualifiedName(services)).ToArray(); + Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName()).ToArray(); Methods = methods.ToArray(); Properties = properties.ToArray(); Fields = fields.ToArray(); @@ -125,9 +125,76 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { GenericBaseParameters = GenericBaseParameters ?? Array.Empty(); GenericParameterValues = cls.GenericParameters - .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName(services) }) + .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName() }) .ToArray(); } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + if(_cls != null) { + return _cls; + } + _cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan())); + var bases = CreateBases(mf, gs); + + _cls.SetBases(bases); + _cls.SetDocumentation(Documentation); + + if (GenericParameterValues.Length > 0) { + _cls.StoreGenericParameters( + _cls, + _cls.GenericParameters.Keys.ToArray(), + GenericParameterValues.ToDictionary( + k => _cls.GenericParameters.Keys.First(x => x == k.Name), + v => mf.ConstructType(v.Type) + ) + ); + } + + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + _cls.AddMember(m.Name, m.Create(mf, _cls, gs), false); + } + return _cls; + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + foreach (var m in all) { + m.Populate(mf, _cls, gs); + } + } + + private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { + var ntBases = NamedTupleBases.Select(ntb => { + var n = ntb.Create(mf, _cls, gs); + ntb.Populate(mf, _cls, gs); + return n; + }).OfType().ToArray(); + + var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); + var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); + + if (GenericBaseParameters.Length > 0) { + // Generic class. Need to reconstruct generic base so code can then + // create specific types off the generic class. + var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); + if (genericBase != null) { + var typeVars = GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); + if (typeVars.Length > 0) { + var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); + if (genericWithParameters != null) { + bases = bases.Except(Enumerable.Repeat(genericBase, 1)).Concat(Enumerable.Repeat(genericWithParameters, 1)).ToArray(); + } + } + } else { + Debug.Fail("Generic class does not have generic base."); + } + } + return bases; + } + protected override IEnumerable GetMemberModels() => Classes.Concat(Methods).Concat(Properties).Concat(Fields); } diff --git a/src/Caching/Impl/Models/DottedNameModel.cs b/src/Caching/Impl/Models/DottedNameModel.cs new file mode 100644 index 000000000..1bbc05822 --- /dev/null +++ b/src/Caching/Impl/Models/DottedNameModel.cs @@ -0,0 +1,23 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] + internal sealed class DottedNameModel { + public string[] NameParts { get; set; } + } +} diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs new file mode 100644 index 000000000..f5a670056 --- /dev/null +++ b/src/Caching/Impl/Models/FromImportModel.cs @@ -0,0 +1,26 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents from import statement for dependency resolution. + /// + internal sealed class FromImportModel { + public string[] RootNames { get; set; } + public string[] MemberNames { get; set; } + public int DotCount { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Caching/Impl/Models/FunctionAttributes.cs b/src/Caching/Impl/Models/FunctionAttributes.cs index 45286ca07..1c0f3da74 100644 --- a/src/Caching/Impl/Models/FunctionAttributes.cs +++ b/src/Caching/Impl/Models/FunctionAttributes.cs @@ -17,7 +17,6 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Flags] - [Serializable] internal enum FunctionAttributes { Normal, Abstract, diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 277330f75..dbfd63ddf 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -17,7 +17,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; +using Microsoft.Python.Analysis.Values; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -29,20 +29,47 @@ internal sealed class FunctionModel : CallableModel { public OverloadModel[] Overloads { get; set; } public FunctionModel() { } // For de-serializer from JSON - public FunctionModel(IPythonFunctionType func, IServiceContainer services) : base(func, services) { - Overloads = func.Overloads.Select(s => FromOverload(s, services)).ToArray(); + [NonSerialized] private PythonFunctionType _function; + + public FunctionModel(IPythonFunctionType func) : base(func) { + Overloads = func.Overloads.Select(FromOverload).ToArray(); } - private static OverloadModel FromOverload(IPythonFunctionOverload o, IServiceContainer services) + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _function ?? (_function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation)); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + // Create inner functions and classes first since function may be returning one of them. + var all = Classes.Concat(Functions).ToArray(); + + foreach (var model in all) { + _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true); + } + foreach (var model in all) { + model.Populate(mf, _function, gs); + } + + foreach (var om in Overloads) { + var o = new PythonFunctionOverload(_function, new Location(mf.Module, IndexSpan.ToSpan())); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray()); + _function.AddOverload(o); + } + } + + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) + => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); + + private static OverloadModel FromOverload(IPythonFunctionOverload o) => new OverloadModel { Parameters = o.Parameters.Select(p => new ParameterModel { Name = p.Name, - Type = p.Type.GetPersistentQualifiedName(services), + Type = p.Type.GetPersistentQualifiedName(), Kind = p.Kind, - DefaultValue = p.DefaultValue.GetPersistentQualifiedName(services), + DefaultValue = p.DefaultValue.GetPersistentQualifiedName(), }).ToArray(), - ReturnType = o.StaticReturnValue.GetPersistentQualifiedName(services), - Documentation = o.Documentation + ReturnType = o.StaticReturnValue.GetPersistentQualifiedName() }; } } diff --git a/src/Caching/Impl/Models/GenericParameterValueModel.cs b/src/Caching/Impl/Models/GenericParameterValueModel.cs index c6287920d..17f0c68f6 100644 --- a/src/Caching/Impl/Models/GenericParameterValueModel.cs +++ b/src/Caching/Impl/Models/GenericParameterValueModel.cs @@ -13,14 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - namespace Microsoft.Python.Analysis.Caching.Models { /// /// Model for actual values assigned to generic parameters. /// I.e. if class is based on Generic[T], what is assigned to T. /// - [Serializable] internal sealed class GenericParameterValueModel { /// /// Generic parameter name as defined by TypeVar, such as T. diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs new file mode 100644 index 000000000..42d479291 --- /dev/null +++ b/src/Caching/Impl/Models/ImportModel.cs @@ -0,0 +1,24 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis.Caching.Models { + /// + /// Represents import statement for dependency resolution. + /// + internal sealed class ImportModel { + public DottedNameModel[] ModuleNames { get; set; } + public bool ForceAbsolute { get; set; } + } +} diff --git a/src/Caching/Impl/Models/IndexSpanModel.cs b/src/Caching/Impl/Models/IndexSpanModel.cs index 08f9719bb..b9ccc45d7 100644 --- a/src/Caching/Impl/Models/IndexSpanModel.cs +++ b/src/Caching/Impl/Models/IndexSpanModel.cs @@ -14,13 +14,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Diagnostics; using Microsoft.Python.Core.Text; // ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { - [Serializable] [DebuggerDisplay("{Start}:({Length})")] internal sealed class IndexSpanModel { public int Start { get; set; } diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index 15a5fba3f..e7ce43282 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -16,6 +16,8 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; // ReSharper disable MemberCanBeProtected.Global // ReSharper disable UnusedAutoPropertyAccessor.Global @@ -32,11 +34,6 @@ internal abstract class MemberModel { /// public string Name { get; set; } - /// - /// Unique id of declaring module. - /// - public string DeclaringModuleId { get; set; } - /// /// Member qualified name within the module, such as A.B.C. /// @@ -47,6 +44,17 @@ internal abstract class MemberModel { /// public IndexSpanModel IndexSpan { get; set; } + /// + /// Create member for declaration but does not construct its parts just yet. + /// Used as a first pass in two-pass handling of forward declarations. + /// + public abstract IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + + /// + /// Populate member with content, such as create class methods, etc. + /// + public abstract void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 929dccbff..8adf55a86 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -21,6 +21,7 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; // ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { @@ -30,15 +31,13 @@ internal sealed class ModuleModel : MemberModel { /// Module unique id that includes version. /// public string UniqueId { get; set; } - public string FilePath { get; set; } + public string Documentation { get; set; } public FunctionModel[] Functions { get; set; } public VariableModel[] Variables { get; set; } public ClassModel[] Classes { get; set; } public TypeVarModel[] TypeVars { get; set; } public NamedTupleModel[] NamedTuples { get; set; } - //public SubmoduleModel[] SubModules { get; set; } - /// /// Collection of new line information for conversion of linear spans /// to line/columns in navigation to member definitions and references. @@ -50,12 +49,13 @@ internal sealed class ModuleModel : MemberModel { /// public int FileSize { get; set; } + public ImportModel[] Imports { get; set; } + public FromImportModel[] FromImports { get; set; } + public ImportModel[] StubImports { get; set; } + public FromImportModel[] StubFromImports { get; set; } + [NonSerialized] private Dictionary _modelCache; - [NonSerialized] private object _modelCacheLock = new object(); - /// - /// Constructs module persistent model from analysis. - /// public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { var uniqueId = analysis.Document.GetUniqueId(services, options); if (uniqueId == null) { @@ -68,19 +68,23 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta var classes = new Dictionary(); var typeVars = new Dictionary(); var namedTuples = new Dictionary(); - //var subModules = new Dictionary(); - - foreach (var v in analysis.Document.GetMemberNames() - .Select(x => analysis.GlobalScope.Variables[x]).ExcludeDefault()) { + // Go directly through variables which names are listed in GetMemberNames + // as well as variables that are declarations. + var exportedNames = new HashSet(analysis.Document.GetMemberNames()); + foreach (var v in analysis.GlobalScope.Variables + .Where(v => exportedNames.Contains(v.Name) || + v.Source == VariableSource.Declaration || + v.Source == VariableSource.Builtin || + v.Source == VariableSource.Generic)) { if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { - typeVars[v.Name] = TypeVarModel.FromGeneric(v, services); + typeVars[v.Name] = TypeVarModel.FromGeneric(v); continue; } switch (v.Value) { case ITypingNamedTupleType nt: - namedTuples[v.Name] = new NamedTupleModel(nt, services); + namedTuples[nt.Name] = new NamedTupleModel(nt); continue; case IPythonFunctionType ft when ft.IsLambda(): // No need to persist lambdas. @@ -91,7 +95,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta // x = type(func) break; case IPythonFunctionType ft: - var fm = GetFunctionModel(analysis, v, ft, services); + var fm = GetFunctionModel(analysis, v, ft); if (fm != null && !functions.ContainsKey(ft.Name)) { functions[ft.Name] = fm; continue; @@ -103,7 +107,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta case IPythonClassType cls when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub): if (!classes.ContainsKey(cls.Name)) { - classes[cls.Name] = new ClassModel(cls, services); + classes[cls.Name] = new ClassModel(cls); continue; } break; @@ -111,62 +115,102 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals // Do not re-declare classes and functions as variables in the model. if (!variables.ContainsKey(v.Name)) { - variables[v.Name] = VariableModel.FromVariable(v, services); + variables[v.Name] = VariableModel.FromVariable(v); } } + // Take dependencies from imports. If module has stub we should also take + // dependencies from there since persistent state is based on types that + // are combination of stub and the module. Sometimes stub may import more + // and we must make sure dependencies are restored before the module. + var primaryDependencyWalker = new DependencyWalker(analysis.Ast); + var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null; + var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty(); + var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty(); + return new ModuleModel { Id = uniqueId.GetStableHash(), UniqueId = uniqueId, Name = analysis.Document.Name, QualifiedName = analysis.Document.QualifiedName, - FilePath = analysis.Document.FilePath, Documentation = analysis.Document.Documentation, Functions = functions.Values.ToArray(), Variables = variables.Values.ToArray(), Classes = classes.Values.ToArray(), TypeVars = typeVars.Values.ToArray(), NamedTuples = namedTuples.Values.ToArray(), - //SubModules = subModules.Values.ToArray(), NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel { EndIndex = l.EndIndex, Kind = l.Kind }).ToArray(), - FileSize = analysis.Ast.EndIndex + FileSize = analysis.Ast.EndIndex, + Imports = primaryDependencyWalker.Imports.ToArray(), + FromImports = primaryDependencyWalker.FromImports.ToArray(), + StubImports = stubImports.ToArray(), + StubFromImports = stubFromImports.ToArray() }; } - private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVariable v, IPythonFunctionType f, IServiceContainer services) { + private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVariable v, IPythonFunctionType f) { if (v.Source == VariableSource.Import && !f.DeclaringModule.Equals(analysis.Document) && !f.DeclaringModule.Equals(analysis.Document.Stub)) { // It may be that the function is from a child module via import. // For example, a number of functions in 'os' are imported from 'nt' on Windows via // star import. Their stubs, however, come from 'os' stub. The function then have declaring // module as 'nt' rather than 'os' and 'nt' does not have a stub. In this case use function // model like if function was declared in 'os'. - return new FunctionModel(f, services); + return new FunctionModel(f); } if (f.DeclaringModule.Equals(analysis.Document) || f.DeclaringModule.Equals(analysis.Document.Stub)) { - return new FunctionModel(f, services); + return new FunctionModel(f); } return null; } - + public override MemberModel GetModel(string name) { - lock (_modelCacheLock) { - if (_modelCache == null) { - _modelCache = new Dictionary(); - foreach (var m in GetMemberModels()) { - Debug.Assert(!_modelCache.ContainsKey(m.Name)); - _modelCache[m.Name] = m; - } + if (_modelCache == null) { + var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); + _modelCache = new Dictionary(); + foreach (var m in models) { + Debug.Assert(!_modelCache.ContainsKey(m.Name)); + _modelCache[m.Name] = m; } - - return _modelCache.TryGetValue(name, out var model) ? model : null; } + return _modelCache.TryGetValue(name, out var model) ? model : null; } - protected override IEnumerable GetMemberModels() - => TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + + private sealed class DependencyWalker : PythonWalker { + public List Imports { get; } = new List(); + public List FromImports { get; } = new List(); + + public DependencyWalker(PythonAst ast) { + ast.Walk(this); + } + + public override bool Walk(ImportStatement import) { + var model = new ImportModel { + ForceAbsolute = import.ForceAbsolute, + ModuleNames = import.Names.Select(mn => new DottedNameModel { + NameParts = mn.Names.Select(nex => nex.Name).ToArray() + }).ToArray() + }; + Imports.Add(model); + return false; + } + + public override bool Walk(FromImportStatement fromImport) { + var model = new FromImportModel { + ForceAbsolute = fromImport.ForceAbsolute, + RootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(), + MemberNames = fromImport.Names.Select(n => n.Name).ToArray(), + DotCount = fromImport.Root is RelativeModuleName rn ? rn.DotCount : 0 + }; + FromImports.Add(model); + return false; + } + } } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index ae473d30a..82fed074b 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -18,6 +18,8 @@ using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Models { @@ -27,16 +29,29 @@ internal sealed class NamedTupleModel: MemberModel { public string[] ItemNames { get; set; } public string[] ItemTypes { get; set; } + [NonSerialized] private NamedTupleType _namedTuple; + public NamedTupleModel() { } // For de-serializer from JSON - public NamedTupleModel(ITypingNamedTupleType nt, IServiceContainer services) { + public NamedTupleModel(ITypingNamedTupleType nt) { Id = nt.Name.GetStableHash(); Name = nt.Name; - DeclaringModuleId = nt.DeclaringModule.GetUniqueId(services); QualifiedName = nt.QualifiedName; IndexSpan = nt.Location.IndexSpan.ToModel(); ItemNames = nt.ItemNames.ToArray(); ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + if (_namedTuple != null) { + return _namedTuple; + } + + var itemTypes = ItemTypes.Select(mf.ConstructType).ToArray(); + _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan()); + return _namedTuple; + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/Models/NewLineModel.cs b/src/Caching/Impl/Models/NewLineModel.cs index 9b4fe0bf5..b3021102d 100644 --- a/src/Caching/Impl/Models/NewLineModel.cs +++ b/src/Caching/Impl/Models/NewLineModel.cs @@ -14,11 +14,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Caching.Models { - [Serializable] internal sealed class NewLineModel { public int EndIndex { get; set; } public NewLineKind Kind { get; set; } diff --git a/src/Caching/Impl/Models/OverloadModel.cs b/src/Caching/Impl/Models/OverloadModel.cs index d5a7d5fa9..1f0014eef 100644 --- a/src/Caching/Impl/Models/OverloadModel.cs +++ b/src/Caching/Impl/Models/OverloadModel.cs @@ -13,13 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - namespace Microsoft.Python.Analysis.Caching.Models { - [Serializable] internal sealed class OverloadModel { public ParameterModel[] Parameters { get; set; } public string ReturnType { get; set; } - public string Documentation { get; set; } } } diff --git a/src/Caching/Impl/Models/ParameterModel.cs b/src/Caching/Impl/Models/ParameterModel.cs index 63e627753..6006b385b 100644 --- a/src/Caching/Impl/Models/ParameterModel.cs +++ b/src/Caching/Impl/Models/ParameterModel.cs @@ -13,11 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching.Models { - [Serializable] internal sealed class ParameterModel { public string Name { get; set; } public string Type { get; set; } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index 6108b1f6c..bea252b81 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -15,7 +15,7 @@ using System; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; +using Microsoft.Python.Analysis.Values; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -24,13 +24,24 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Serializable] internal sealed class PropertyModel : CallableModel { public string ReturnType { get; set; } - public bool IsReadOnly { get; set; } - public PropertyModel() { } // For de-serializer from JSON - public PropertyModel(IPythonPropertyType prop, IServiceContainer services) : base(prop, services) { - ReturnType = prop.ReturnType.GetPersistentQualifiedName(services); - IsReadOnly = prop.IsReadOnly; + [NonSerialized] private PythonPropertyType _property; + + public PropertyModel(IPythonPropertyType prop) : base(prop) { + ReturnType = prop.ReturnType.GetPersistentQualifiedName(); + } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => _property ?? (_property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), Documentation, declaringType, (Attributes & FunctionAttributes.Abstract) != 0)); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + _property.SetDocumentation(Documentation); + + var o = new PythonFunctionOverload(_property, mf.DefaultLocation); + o.SetDocumentation(Documentation); + o.SetReturnValue(mf.ConstructMember(ReturnType), true); + _property.AddOverload(o); } } } diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index 6665a9957..e630d801f 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -17,6 +17,8 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; // ReSharper disable MemberCanBePrivate.Global @@ -30,17 +32,24 @@ internal sealed class TypeVarModel : MemberModel { public object Covariant { get; set; } public object Contravariant { get; set; } - public static TypeVarModel FromGeneric(IVariable v, IServiceContainer services) { + public static TypeVarModel FromGeneric(IVariable v) { var g = (IGenericTypeParameter)v.Value; return new TypeVarModel { Id = g.Name.GetStableHash(), Name = g.Name, QualifiedName = g.QualifiedName, - Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName(services)).ToArray(), + Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName()).ToArray(), Bound = g.Bound?.QualifiedName, Covariant = g.Covariant, Contravariant = g.Contravariant }; } + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) + => new GenericTypeParameter(Name, mf.Module, + Constraints.Select(mf.ConstructType).ToArray(), + mf.ConstructType(Bound), Covariant, Contravariant, default); + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index 6aecbcda4..892dac8ac 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -26,27 +26,34 @@ namespace Microsoft.Python.Analysis.Caching.Models { internal sealed class VariableModel : MemberModel { public string Value { get; set; } - public static VariableModel FromVariable(IVariable v, IServiceContainer services) => new VariableModel { + public static VariableModel FromVariable(IVariable v) => new VariableModel { Id = v.Name.GetStableHash(), Name = v.Name, QualifiedName = v.Name, IndexSpan = v.Location.IndexSpan.ToModel(), - Value = v.Value.GetPersistentQualifiedName(services) + Value = v.Value.GetPersistentQualifiedName() }; - public static VariableModel FromInstance(string name, IPythonInstance inst, IServiceContainer services) => new VariableModel { + public static VariableModel FromInstance(string name, IPythonInstance inst) => new VariableModel { Id = name.GetStableHash(), Name = name, QualifiedName = name, - Value = inst.GetPersistentQualifiedName(services) + Value = inst.GetPersistentQualifiedName() }; - public static VariableModel FromType(string name, IPythonType t, IServiceContainer services) => new VariableModel { + public static VariableModel FromType(string name, IPythonType t) => new VariableModel { Id = name.GetStableHash(), Name = name, QualifiedName = name, IndexSpan = t.Location.IndexSpan.ToModel(), - Value = t.GetPersistentQualifiedName(services) + Value = t.GetPersistentQualifiedName() }; + + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType; + return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default)); + } + + public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 2e51aedde..75f423834 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -14,120 +14,157 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using LiteDB; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Caching.IO; using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.Services; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { - internal sealed class ModuleDatabase : IModuleDatabaseService, IDisposable { - private readonly object _modulesLock = new object(); - private readonly Dictionary _modulesCache - = new Dictionary(); - - private readonly ConcurrentDictionary _modelsCache - = new ConcurrentDictionary(); - - private readonly ConcurrentDictionary _searchResults - = new ConcurrentDictionary(); + internal sealed class ModuleDatabase : IModuleDatabaseService { + private readonly Dictionary _dependencies = new Dictionary(); + private readonly object _lock = new object(); private readonly IServiceContainer _services; private readonly ILogger _log; private readonly IFileSystem _fs; - private readonly AnalysisCachingLevel _defaultCachingLevel; - private readonly CacheWriter _cacheWriter; - private AnalysisCachingLevel? _cachingLevel; - - public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { - _services = sm; - _log = _services.GetService(); - _fs = _services.GetService(); - _defaultCachingLevel = AnalysisCachingLevel.Library; - var cfs = _services.GetService(); - CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); - _cacheWriter = new CacheWriter(_services.GetService(), _fs, _log, CacheFolder); - sm.AddService(this); + + public ModuleDatabase(IServiceContainer services) { + _services = services; + _log = services.GetService(); + _fs = services.GetService(); + + var cfs = services.GetService(); + CacheFolder = Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); } public string CacheFolderBaseName => "analysis.v"; - public int DatabaseFormatVersion => 5; + public int DatabaseFormatVersion => 2; public string CacheFolder { get; } + /// + /// Retrieves dependencies from the module persistent state. + /// + /// Python module to restore analysis for. + /// Python module dependency provider. + public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) { + dp = null; + + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; + } + + lock (_lock) { + if (_dependencies.TryGetValue(module.Name, out dp)) { + return true; + } + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + dp = new DependencyProvider(module, model); + _dependencies[module.Name] = dp; + return true; + } + } + return false; + } + /// /// Creates global scope from module persistent state. /// Global scope is then can be used to construct module analysis. /// - public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType) { - if (GetCachingLevel() == AnalysisCachingLevel.None) { - return null; + /// Python module to restore analysis for. + /// Python module global scope. + public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) { + gs = null; + + if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { + return false; + } + + lock (_lock) { + if (FindModuleModel(module.Name, module.FilePath, out var model)) { + gs = new RestoredGlobalScope(model, module); + } } - return FindModuleModelByPath(moduleName, modulePath, moduleType, out var model) - ? RestoreModule(model) : null; + + return gs != null; } + /// + /// Writes module data to the database. + /// + public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) + => Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken), cancellationToken); + /// /// Determines if module analysis exists in the storage. /// - public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { + public bool ModuleExistsInStorage(string moduleName, string filePath) { if (GetCachingLevel() == AnalysisCachingLevel.None) { return false; } - var key = new AnalysisModuleKey(name, filePath); - if (_searchResults.TryGetValue(key, out var result)) { - return result; + for (var retries = 50; retries > 0; --retries) { + try { + var dbPath = FindDatabaseFile(moduleName, filePath); + return !string.IsNullOrEmpty(dbPath); + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { + Thread.Sleep(10); + } } - - WithRetries.Execute(() => { - var dbPath = FindDatabaseFile(name, filePath, moduleType); - _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); - return result; - }, "Unable to find database file for {name}.", _log); return false; } - public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default) { + private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); if (cachingLevel == AnalysisCachingLevel.None) { return; } - var model = await Task.Run(() => ModuleModel.FromAnalysis(analysis, _services, cachingLevel), cancellationToken); - if (model != null && !cancellationToken.IsCancellationRequested) { - await _cacheWriter.EnqueueModel(model, immediate, cancellationToken); + var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); + if (model == null) { + // Caching level setting does not permit this module to be persisted. + return; } - } - internal IPythonModule RestoreModule(string moduleName, string uniqueId) { - lock (_modulesLock) { - if (_modulesCache.TryGetValue(uniqueId, out var m)) { - return m; + Exception ex = null; + for (var retries = 50; retries > 0; --retries) { + cancellationToken.ThrowIfCancellationRequested(); + try { + if (!_fs.DirectoryExists(CacheFolder)) { + _fs.CreateDirectory(CacheFolder); + } + + cancellationToken.ThrowIfCancellationRequested(); + using (var db = new LiteDatabase(Path.Combine(CacheFolder, $"{model.UniqueId}.db"))) { + var modules = db.GetCollection("modules"); + modules.Upsert(model); + return; + } + } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { + ex = ex1; + Thread.Sleep(10); + } catch (Exception ex2) { + ex = ex2; + break; } } - return FindModuleModelById(moduleName, uniqueId, out var model) ? RestoreModule(model) : null; - } - private IPythonModule RestoreModule(ModuleModel model) { - PythonDbModule dbModule; - lock (_modulesLock) { - if (_modulesCache.TryGetValue(model.UniqueId, out var m)) { - return m; + if (ex != null) { + _log?.Log(System.Diagnostics.TraceEventType.Warning, $"Unable to write analysis of {model.Name} to database. Exception {ex.Message}"); + if (ex.IsCriticalException()) { + throw ex; } - dbModule = _modulesCache[model.UniqueId] = new PythonDbModule(model, model.FilePath, _services); } - dbModule.Construct(model); - return dbModule; } /// @@ -135,12 +172,13 @@ private IPythonModule RestoreModule(ModuleModel model) { /// by name, version, current Python interpreter version and/or hash of the /// module content (typically file sizes). /// - private string FindDatabaseFile(string moduleName, string filePath, ModuleType moduleType) { - var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, GetCachingLevel()); - return string.IsNullOrEmpty(uniqueId) ? null : FindDatabaseFile(uniqueId); - } + private string FindDatabaseFile(string moduleName, string filePath) { + var interpreter = _services.GetService(); + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel()); + if (string.IsNullOrEmpty(uniqueId)) { + return null; + } - private string FindDatabaseFile(string uniqueId) { // Try module name as is. var dbPath = Path.Combine(CacheFolder, $"{uniqueId}.db"); if (_fs.FileExists(dbPath)) { @@ -149,7 +187,6 @@ private string FindDatabaseFile(string uniqueId) { // TODO: resolving to a different version can be an option // Try with the major.minor Python version. - var interpreter = _services.GetService(); var pythonVersion = interpreter.Configuration.Version; dbPath = Path.Combine(CacheFolder, $"{uniqueId}({pythonVersion.Major}.{pythonVersion.Minor}).db"); @@ -162,40 +199,48 @@ private string FindDatabaseFile(string uniqueId) { return _fs.FileExists(dbPath) ? dbPath : null; } - private bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) - => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, modulePath, moduleType), out model); - - private bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) - => TryGetModuleModel(moduleName, FindDatabaseFile(uniqueId), out model); - - private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { + private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) { model = null; - if (string.IsNullOrEmpty(dbPath)) { - return false; + // We don't cache results here. Module resolution service decides when to call in here + // and it is responsible of overall management of the loaded Python modules. + for (var retries = 50; retries > 0; --retries) { + try { + // TODO: make combined db rather than per module? + var dbPath = FindDatabaseFile(moduleName, filePath); + if (string.IsNullOrEmpty(dbPath)) { + return false; + } + + using (var db = new LiteDatabase(dbPath)) { + if (!db.CollectionExists("modules")) { + return false; + } + + var modules = db.GetCollection("modules"); + model = modules.Find(m => m.Name == moduleName).FirstOrDefault(); + return model != null; + } + } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { + Thread.Sleep(10); + } } + return false; + } + private AnalysisCachingLevel GetCachingLevel() + => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; - if (_modelsCache.TryGetValue(moduleName, out model)) { - return true; - } + private sealed class DependencyProvider : IDependencyProvider { + private readonly ISet _dependencies; - model = WithRetries.Execute(() => { - using (var db = new LiteDatabase(dbPath)) { - var modules = db.GetCollection("modules"); - var storedModel = modules.FindOne(m => m.Name == moduleName); - _modelsCache[moduleName] = storedModel; - return storedModel; - } - }, $"Unable to locate database for module {moduleName}.", _log); + public DependencyProvider(IPythonModule module, ModuleModel model) { + var dc = new DependencyCollector(module); + dc.AddImports(model.Imports); + dc.AddFromImports(model.FromImports); + _dependencies = dc.Dependencies; + } - return model != null; + public ISet GetDependencies(PythonAst ast) => _dependencies; } - - private AnalysisCachingLevel GetCachingLevel() - => _cachingLevel - ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) - ?? _defaultCachingLevel; - - public void Dispose() => _cacheWriter.Dispose(); } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 28e6a94b2..c9794a714 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -17,13 +17,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Microsoft.Python.Analysis.Caching.Lazy; using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -35,20 +34,18 @@ internal sealed class ModuleFactory { /// For use in tests so missing members will assert. internal static bool EnableMissingMemberAssertions { get; set; } + // TODO: better resolve circular references. + private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); private readonly ModuleModel _model; private readonly IGlobalScope _gs; - private readonly ModuleDatabase _db; - private readonly IServiceContainer _services; public IPythonModule Module { get; } public Location DefaultLocation { get; } - public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, IServiceContainer services) { - _model = model ?? throw new ArgumentNullException(nameof(model)); - _gs = gs ?? throw new ArgumentNullException(nameof(gs)); - _services = services ?? throw new ArgumentNullException(nameof(services)); - _db = services.GetService(); - Module = module ?? throw new ArgumentNullException(nameof(module)); + public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) { + _model = model; + _gs = gs; + Module = module; DefaultLocation = new Location(Module); } @@ -99,17 +96,17 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } if (memberName == "") { - return new PythonFunctionType("", default, default, string.Empty); + return null; } var nextModel = currentModel.GetModel(memberName); - //Debug.Assert(nextModel != null, - // $"Unable to find {string.Join(".", memberNames)} in module {Module.Name}"); + Debug.Assert(nextModel != null, + $"Unable to find {string.Join(".", memberNames)} in module {Module.Name}"); if (nextModel == null) { return null; } - m = MemberFactory.CreateMember(nextModel, this, _gs, declaringType); + m = nextModel.Create(this, declaringType, _gs); Debug.Assert(m != null); if (m is IGenericType gt && typeArgs.Count > 0) { @@ -117,8 +114,11 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } currentModel = nextModel; - declaringType = m.GetPythonType(); + declaringType = m as IPythonType; Debug.Assert(declaringType != null); + if (declaringType == null) { + return null; + } } return m; @@ -132,44 +132,25 @@ private IPythonModule GetModule(QualifiedNameParts parts) { if (parts.ModuleName == Module.Name) { return Module; } - - // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. - // If module is a stub, first try regular module, then the stub since with regular modules - // stub data is merged into the module data but there are also standalone stubs like posix. - var mres = Module.Interpreter.ModuleResolution; - var tres = Module.Interpreter.TypeshedResolution; - - var module = mres.GetImportedModule(parts.ModuleName); - if (module == null && parts.IsStub) { - module = tres.GetImportedModule(parts.ModuleName); - } - - // If module is not loaded, try database. - if (module == null) { - var moduleId = parts.ModuleId ?? parts.ModuleName; - module = _db?.RestoreModule(parts.ModuleName, moduleId); - } - - if (module == null) { - // Fallback if somehow module is not loaded or missing from the database. - // Try loading it directly and wait a bit hoping for the analysis to complete. - var resolution = parts.IsStub ? tres : mres; - var imports = resolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, Enumerable.Repeat(parts.ModuleName, 1), true); - if (imports is ModuleImport moduleImport) { - module = resolution.GetOrLoadModule(moduleImport.FullName); + using (_moduleReentrancy.Push(parts.ModuleName, out var reentered)) { + if (reentered) { + return null; } + // Here we do not call GetOrLoad since modules references here must + // either be loaded already since they were required to create + // persistent state from analysis. Also, occasionally types come + // from the stub and the main module was never loaded. This, for example, + // happens with io which has member with mmap type coming from mmap + // stub rather than the primary mmap module. + var m = parts.IsStub + ? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName) + : Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); + + if (m != null) { + return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(m) : m; + } + return null; } - - // Here we do not call GetOrLoad since modules references here must - // either be loaded already since they were required to create - // persistent state from analysis. Also, occasionally types come - // from the stub and the main module was never loaded. This, for example, - // happens with io which has member with mmap type coming from mmap - // stub rather than the primary mmap module. - if (module != null) { - module = parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; - } - return module; } private IMember GetMember(IMember root, IEnumerable memberNames) { @@ -200,12 +181,12 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { } if (member == null) { - //var containerName = mc is IPythonType t ? t.Name : ""; - //Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); + var containerName = mc is IPythonType t ? t.Name : ""; + Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); break; } - member = typeArgs.Count > 0 && member is IGenericType gt && typeArgs.Any(a => !(a is IGenericTypeParameter)) + member = typeArgs.Count > 0 && member is IGenericType gt ? gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)) : member; } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 932c800e2..51762ecfe 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -27,36 +26,16 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { - private struct ModuleKey { - public string ModuleName { get; set; } - public string FilePath { get; set; } - public ModuleType ModuleType { get; set; } - } - - private static readonly ConcurrentDictionary _nameCache = new ConcurrentDictionary(); + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel) + => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); - public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { - // If module is a standalone stub, permit it. Otherwise redirect to the main module - // since during stub merge types from stub normally become part of the primary module. - if (module.ModuleType == ModuleType.Stub && module.PrimaryModule != null) { - module = module.PrimaryModule; - } - return GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); - } - - public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, - IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { - if (cachingLevel == AnalysisCachingLevel.None) { + public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel) { + if(cachingLevel == AnalysisCachingLevel.None) { return null; } - if (moduleType == ModuleType.User) { - return moduleName; // For tests where user modules are cached. - } - - var key = new ModuleKey { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType }; - if (_nameCache.TryGetValue(key, out var id)) { - return id; + // Only for tests. + return $"{moduleName}"; } var interpreter = services.GetService(); @@ -64,7 +43,7 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType var moduleResolution = interpreter.ModuleResolution; var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs); - switch (modulePathType) { + switch(modulePathType) { case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library: return null; case PythonLibraryPathType.StdLib when cachingLevel < AnalysisCachingLevel.System: @@ -91,37 +70,23 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType if (folders.Length == 1) { var fileName = Path.GetFileNameWithoutExtension(folders[0]); var dash = fileName.IndexOf('-'); - id = $"{moduleName}({fileName.Substring(dash + 1)})"; - break; + return $"{moduleName}({fileName.Substring(dash + 1)})"; } // Move up if nothing is found. versionFolder = Path.GetDirectoryName(versionFolder); } } - if (id == null) { - var config = interpreter.Configuration; - if (moduleType == ModuleType.CompiledBuiltin || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) { - // If module is a standard library, unique id is its name + interpreter version. - id = $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; - } - } - - if (id == null) { - var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); - if (parent == null) { - id = moduleName; - } else { - var hash = HashModuleFileSizes(parent); - // If all else fails, hash modules file sizes. - id = $"{moduleName}.{(ulong)hash}"; - } + var config = interpreter.Configuration; + if (moduleType.IsCompiled() || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) { + // If module is a standard library, unique id is its name + interpreter version. + return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; } - if (id != null) { - _nameCache[key] = id; - } - return id; + var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); + var hash = HashModuleFileSizes(parent); + // If all else fails, hash modules file sizes. + return $"{moduleName}.{(ulong)hash}"; } private static long HashModuleFileSizes(IImportChildrenSource source) { @@ -130,9 +95,6 @@ private static long HashModuleFileSizes(IImportChildrenSource source) { foreach (var name in names) { if (source.TryGetChildImport(name, out var child)) { if (child is ModuleImport moduleImport) { - if (moduleImport.ModuleFileSize == 0) { - continue; // Typically test case, memory-only module. - } hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); } diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index 471871ef8..8da9634ed 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; @@ -32,20 +33,16 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv _fileSize = model.FileSize; } - /// - /// Constructs module global scope. This is separate from regular constructor - /// in order to better handle reentrancy due to circular references - /// in the module factory. - /// public void Construct(ModuleModel model) { - var rs = new RestoredGlobalScope(model, this, Services); - GlobalScope = rs; - rs.Construct(model, Services); + var gs = new RestoredGlobalScope(model, this); + GlobalScope = gs; + gs.ReconstructVariables(); } protected override string LoadContent() => string.Empty; public override string Documentation { get; } + public override IEnumerable GetMemberNames() => GlobalScope.Variables.Names; #region ILocationConverter public override SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(_newLines, index); diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs index e2db7130a..4d1d33539 100644 --- a/src/Caching/Impl/QualifiedNameParts.cs +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -31,8 +31,6 @@ internal struct QualifiedNameParts { public string ModuleName; /// Indicates if module is a stub. public bool IsStub; - /// Module unique id. - public string ModuleId; /// Module member names such as 'A', 'B', 'C' from module:A.B.C. public IReadOnlyList MemberNames; } diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index 4bcf43e20..77e3bb255 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -16,56 +16,62 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Caching.Lazy; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { - internal sealed class RestoredGlobalScope : IGlobalScope { + internal sealed class RestoredGlobalScope : IRestoredGlobalScope { private readonly VariableCollection _scopeVariables = new VariableCollection(); + private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. + private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. - public RestoredGlobalScope(ModuleModel model, IPythonModule module, IServiceContainer services) { + public RestoredGlobalScope(ModuleModel model, IPythonModule module) { + _model = model ?? throw new ArgumentNullException(nameof(model)); Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; + _factory = new ModuleFactory(_model, Module, this); + DeclareVariables(); } - public void Construct(ModuleModel model, IServiceContainer services) { + public void ReconstructVariables() { + var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + foreach (var m in models.Concat(_model.Variables)) { + m.Populate(_factory, null, this); + } + // TODO: re-declare __doc__, __name__, etc. +#if !DEBUG + _model = null; + _factory = null; +#endif + } + + private void DeclareVariables() { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. - var mf = new ModuleFactory(model, Module, this, services); + var mf = new ModuleFactory(_model, Module, this); // Generics first - foreach (var m in model.TypeVars) { - var member = MemberFactory.CreateMember(m, mf, this, null); - _scopeVariables.DeclareVariable(m.Name, member, VariableSource.Generic, mf.DefaultLocation); - } - - var models = model.NamedTuples - .Concat(model.Classes).Concat(model.Functions); //.Concat(_model.SubModules); - foreach (var m in models) { - var member = MemberFactory.CreateMember(m, mf, this, null); - _scopeVariables.DeclareVariable(m.Name, member, VariableSource.Declaration, mf.DefaultLocation); + var typeVars = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); + foreach (var m in typeVars) { + _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation); } - // Now variables in the order of appearance since later variables + // Declare variables in the order of appearance since later variables // may use types declared in the preceding ones. - foreach (var vm in model.Variables.OrderBy(m => m.IndexSpan.Start)) { - var member = MemberFactory.CreateMember(vm, mf, this, null); - _scopeVariables.DeclareVariable(vm.Name, member, VariableSource.Declaration, mf.DefaultLocation); + foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { + var v = (IVariable)vm.Create(mf, null, this); + _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); } } #region IScope public string Name { get; } - public PythonAst Ast => null; public ScopeStatement Node => null; public IScope OuterScope => null; public IReadOnlyList Children => Array.Empty(); - public IScope GetChildScope(ScopeStatement node) => null; public IEnumerable EnumerateTowardsGlobal => Enumerable.Empty(); public IEnumerable EnumerateFromGlobal => Enumerable.Empty(); public IVariableCollection Variables => _scopeVariables; diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs index a1fd1aa7b..6927e2130 100644 --- a/src/Caching/Impl/TypeNames.cs +++ b/src/Caching/Impl/TypeNames.cs @@ -19,7 +19,6 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching { internal static class TypeNames { @@ -27,36 +26,27 @@ internal static class TypeNames { /// Constructs persistent member name based on the member and the current module. /// Persistent name contains complete information for the member restoration code. /// - public static string GetPersistentQualifiedName(this IMember m, IServiceContainer services) { + public static string GetPersistentQualifiedName(this IMember m) { var t = m.GetPythonType(); - string name = null; if (!t.IsUnknown()) { switch (m) { case IPythonInstance _: // constants and strings map here. - name = $"i:{t.QualifiedName}"; - break; + return $"i:{t.QualifiedName}"; case IBuiltinsPythonModule b: return $"b:{b.QualifiedName}"; case PythonVariableModule vm: - name = $"p:{vm.QualifiedName}"; - break; + return $"p:{vm.QualifiedName}"; case IPythonModule mod: - name = $"m:{mod.QualifiedName}"; - break; + return $"m:{mod.QualifiedName}"; case IPythonType pt when pt.DeclaringModule.ModuleType == ModuleType.Builtins: return $"t:{(pt.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : pt.QualifiedName)}"; case IPythonType pt: - name = $"t:{pt.QualifiedName}"; - break; + return $"t:{pt.QualifiedName}"; case null: break; } } - - if (name == null || t.DeclaringModule.ModuleType == ModuleType.Builtins) { - return name; - } - return $"{name}${t.DeclaringModule.GetUniqueId(services)}"; + return null; } /// @@ -71,12 +61,6 @@ public static bool DeconstructQualifiedName(string qualifiedName, out QualifiedN return false; } - var index = qualifiedName.IndexOf('$'); - if (index > 0) { - parts.ModuleId = qualifiedName.Substring(index + 1); - qualifiedName = qualifiedName.Substring(0, index); - } - GetObjectTypeFromPrefix(qualifiedName, ref parts, out var prefixOffset); GetModuleNameAndMembers(qualifiedName, ref parts, prefixOffset); @@ -118,6 +102,7 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN default: parts.ModuleName = typeName; parts.MemberNames = Array.Empty(); + DetermineModuleType(ref parts); break; } return; diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 9ebca86f6..8baa49828 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -27,7 +27,7 @@ using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { - public abstract class AnalysisCachingTestBase : AnalysisTestBase { + public abstract class AnalysisCachingTestBase: AnalysisTestBase { protected AnalysisCachingTestBase() { ModuleFactory.EnableMissingMemberAssertions = true; } @@ -51,8 +51,8 @@ protected string BaselineFilesFolder { } } - protected string GetBaselineFileName(string testName, string suffix = null) - => Path.ChangeExtension(suffix == null + protected string GetBaselineFileName(string testName, string suffix = null) + => Path.ChangeExtension(suffix == null ? Path.Combine(BaselineFilesFolder, testName) : Path.Combine(BaselineFilesFolder, testName + suffix), "json"); @@ -62,20 +62,33 @@ internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { return dbModule; } - internal async Task CompareRestoreAsync(ModuleModel model, IPythonModule m, bool recursive = false) { + internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + + // In real case dependency analysis will restore model dependencies. + // Here we don't go through the dependency analysis so we have to + // manually restore dependent modules. + var dc = new DependencyCollector(m); + dc.AddImports(model.Imports); + dc.AddFromImports(model.FromImports); + foreach(var dep in dc.Dependencies) { + m.Interpreter.ModuleResolution.GetOrLoadModule(dep.Name); + } + + var dcs = new DependencyCollector(m, true); + dcs.AddImports(model.StubImports); + dcs.AddFromImports(model.StubFromImports); + foreach (var dep in dcs.Dependencies) { + m.Interpreter.TypeshedResolution.GetOrLoadModule(dep.Name); + } + var analyzer = Services.GetService(); await analyzer.WaitForCompleteAnalysisAsync(); using (var dbModule = CreateDbModule(model, m.FilePath)) { - dbModule.Should().HaveSameMembersAs(m, recursive); + dbModule.Should().HaveSameMembersAs(m); } } - - internal async Task GetModelAsync(string code) { - var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - model.FilePath = null; - return model; - } } } diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs index 2a520a510..00aeaa9a3 100644 --- a/src/Caching/Test/ClassesTests.cs +++ b/src/Caching/Test/ClassesTests.cs @@ -59,7 +59,8 @@ def methodB2(self): c = B().methodB1() "; - var model = await GetModelAsync(code); + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } @@ -79,7 +80,7 @@ def _methodB(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - await CompareRestoreAsync(model, analysis.Document); + await CompareBaselineAndRestoreAsync(model, analysis.Document); } [TestMethod, Priority(0)] @@ -115,6 +116,9 @@ def func(): analysis.Should().HaveVariable("b").Which.Should().HaveType("B"); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); + using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); } @@ -139,6 +143,8 @@ def value(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); @@ -156,7 +162,8 @@ def __init__(self): '''__init__ doc''' return "; - var model = await GetModelAsync(code); + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); // In JSON, class A should have 'class A doc' documentation while B should have none. Baseline.CompareToFile(BaselineFileName, json); diff --git a/src/Caching/Test/CoreTests.cs b/src/Caching/Test/CoreTests.cs index a91845f56..f1bea1eca 100644 --- a/src/Caching/Test/CoreTests.cs +++ b/src/Caching/Test/CoreTests.cs @@ -58,7 +58,8 @@ def func(): c = C() "; - var model = await GetModelAsync(code); + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } @@ -106,7 +107,6 @@ def func(a): ... .Which.Should().HaveParameters(is3x ? new[] { "a", "b", "c" } : new[] { "a" }); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - model.FilePath = null; var json = ToJson(model); Baseline.CompareToFile(GetBaselineFileNameWithSuffix(is3x ? "3" : "2"), json); } @@ -135,6 +135,8 @@ def func(): "; var analysis = await GetAnalysisAsync(code, PythonVersions.Required_Python38X); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json index 67b5f85b9..647537712 100644 --- a/src/Caching/Test/Files/ClassOwnDocumentation.json +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -1,6 +1,5 @@ { "UniqueId": "module", - "FilePath": null, "Documentation": "", "Functions": [], "Variables": [ @@ -8,7 +7,6 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", - "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -19,7 +17,6 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", - "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -30,7 +27,6 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", - "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -41,7 +37,6 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", - "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -52,7 +47,6 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", - "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -63,7 +57,6 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", - "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -74,7 +67,6 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", - "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -85,7 +77,6 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", - "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -108,7 +99,6 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", - "DeclaringModuleId": "module", "QualifiedName": "module:A", "IndexSpan": { "Start": 8, @@ -116,9 +106,9 @@ } }, { - "Documentation": "class A doc", + "Documentation": null, "Bases": [ - "t:module:A$module", + "t:module:A", "t:object" ], "NamedTupleBases": [], @@ -129,13 +119,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module", + "Type": "t:module:B", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null, - "Documentation": null + "ReturnType": null } ], "Documentation": null, @@ -144,7 +133,6 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module", "QualifiedName": "module:B.__init__", "IndexSpan": { "Start": 58, @@ -159,7 +147,6 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", - "DeclaringModuleId": "module", "QualifiedName": "module:B", "IndexSpan": { "Start": 43, @@ -204,9 +191,12 @@ } ], "FileSize": 115, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index 2a19e01a7..efb7c122d 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -1,6 +1,5 @@ { "UniqueId": "module", - "FilePath": null, "Documentation": "", "Functions": [], "Variables": [ @@ -8,7 +7,6 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", - "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -19,7 +17,6 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", - "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -30,7 +27,6 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", - "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -41,7 +37,6 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", - "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -52,7 +47,6 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", - "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -63,7 +57,6 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", - "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -74,7 +67,6 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", - "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -85,7 +77,6 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", - "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -96,7 +87,6 @@ "Value": "i:str", "Id": 833, "Name": "x", - "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": { "Start": 2, @@ -104,10 +94,9 @@ } }, { - "Value": "i:module:B.C$module", + "Value": "i:module:B.C", "Id": 812, "Name": "c", - "DeclaringModuleId": null, "QualifiedName": "c", "IndexSpan": { "Start": 333, @@ -129,13 +118,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:A$module", + "Type": "t:module:A", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:bool", - "Documentation": null + "ReturnType": "i:bool" } ], "Documentation": null, @@ -144,7 +132,6 @@ "Functions": [], "Id": -1909501047, "Name": "methodA", - "DeclaringModuleId": "module", "QualifiedName": "module:A.methodA", "IndexSpan": { "Start": 33, @@ -159,7 +146,6 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", - "DeclaringModuleId": "module", "QualifiedName": "module:A", "IndexSpan": { "Start": 21, @@ -179,13 +165,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module", + "Type": "t:module:B", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:module:B.C$module", - "Documentation": null + "ReturnType": "i:module:B.C" } ], "Documentation": null, @@ -194,7 +179,6 @@ "Functions": [], "Id": 935009767, "Name": "methodB1", - "DeclaringModuleId": "module", "QualifiedName": "module:B.methodB1", "IndexSpan": { "Start": 235, @@ -207,13 +191,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module", + "Type": "t:module:B", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:int", - "Documentation": null + "ReturnType": "i:int" } ], "Documentation": null, @@ -222,7 +205,6 @@ "Functions": [], "Id": 935009768, "Name": "methodB2", - "DeclaringModuleId": "module", "QualifiedName": "module:B.methodB2", "IndexSpan": { "Start": 287, @@ -236,7 +218,6 @@ "Value": "i:int", "Id": 833, "Name": "x", - "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": null } @@ -255,13 +236,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C$module", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null, - "Documentation": null + "ReturnType": null } ], "Documentation": null, @@ -270,7 +250,6 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module", "QualifiedName": "module:B.C.__init__", "IndexSpan": { "Start": 122, @@ -283,13 +262,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C$module", + "Type": "t:module:B.C", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:bool", - "Documentation": null + "ReturnType": "i:bool" } ], "Documentation": null, @@ -298,7 +276,6 @@ "Functions": [], "Id": -1909501045, "Name": "methodC", - "DeclaringModuleId": "module", "QualifiedName": "module:B.C.methodC", "IndexSpan": { "Start": 175, @@ -312,7 +289,6 @@ "Value": "i:int", "Id": 834, "Name": "y", - "DeclaringModuleId": null, "QualifiedName": "y", "IndexSpan": null } @@ -322,7 +298,6 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", - "DeclaringModuleId": "module", "QualifiedName": "module:B.C", "IndexSpan": { "Start": 106, @@ -334,7 +309,6 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", - "DeclaringModuleId": "module", "QualifiedName": "module:B", "IndexSpan": { "Start": 78, @@ -439,9 +413,12 @@ } ], "FileSize": 353, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index 335855a62..3c94c2dc5 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -1,14 +1,12 @@ { "UniqueId": "module", - "FilePath": null, "Documentation": "", "Functions": [ { "Overloads": [ { "Parameters": [], - "ReturnType": "i:float", - "Documentation": null + "ReturnType": "i:float" } ], "Documentation": null, @@ -17,7 +15,6 @@ "Functions": [], "Id": 24395611, "Name": "func", - "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 207, @@ -30,7 +27,6 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", - "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -41,7 +37,6 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", - "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -52,7 +47,6 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", - "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -63,7 +57,6 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", - "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -74,7 +67,6 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", - "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -85,7 +77,6 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", - "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -96,7 +87,6 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", - "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -107,7 +97,6 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", - "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -118,7 +107,6 @@ "Value": "i:str", "Id": 833, "Name": "x", - "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": { "Start": 2, @@ -126,10 +114,9 @@ } }, { - "Value": "i:module:C$module", + "Value": "i:module:C", "Id": 812, "Name": "c", - "DeclaringModuleId": null, "QualifiedName": "c", "IndexSpan": { "Start": 234, @@ -151,13 +138,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C$module", + "Type": "t:module:C", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null, - "Documentation": null + "ReturnType": null } ], "Documentation": null, @@ -166,7 +152,6 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module", "QualifiedName": "module:C.__init__", "IndexSpan": { "Start": 45, @@ -179,13 +164,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C$module", + "Type": "t:module:C", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:float", - "Documentation": null + "ReturnType": "i:float" } ], "Documentation": null, @@ -194,7 +178,6 @@ "Functions": [], "Id": -2139806792, "Name": "method", - "DeclaringModuleId": "module", "QualifiedName": "module:C.method", "IndexSpan": { "Start": 100, @@ -205,14 +188,12 @@ "Properties": [ { "ReturnType": "i:int", - "IsReadOnly": true, "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 24690682, "Name": "prop", - "DeclaringModuleId": "module", "QualifiedName": "module:C.prop", "IndexSpan": { "Start": 163, @@ -225,7 +206,6 @@ "Value": "i:int", "Id": 833, "Name": "x", - "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": null }, @@ -233,7 +213,6 @@ "Value": "i:int", "Id": 834, "Name": "y", - "DeclaringModuleId": null, "QualifiedName": "y", "IndexSpan": null } @@ -243,7 +222,6 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", - "DeclaringModuleId": "module", "QualifiedName": "module:C", "IndexSpan": { "Start": 21, @@ -332,9 +310,12 @@ } ], "FileSize": 243, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json index 9fd77e9f3..ee4d02e74 100644 --- a/src/Caching/Test/Files/VersionHandling2.json +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -1,6 +1,5 @@ { "UniqueId": "module", - "FilePath": null, "Documentation": "", "Functions": [ { @@ -14,8 +13,7 @@ "Kind": 0 } ], - "ReturnType": null, - "Documentation": null + "ReturnType": null } ], "Documentation": null, @@ -24,7 +22,6 @@ "Functions": [], "Id": 24395611, "Name": "func", - "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 77, @@ -37,7 +34,6 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", - "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -48,7 +44,6 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", - "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -59,7 +54,6 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", - "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -70,7 +64,6 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", - "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -81,7 +74,6 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", - "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -92,7 +84,6 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", - "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -103,7 +94,6 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", - "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -114,7 +104,6 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", - "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -148,9 +137,12 @@ } ], "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index d3e874afa..1d92c7ff2 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -1,6 +1,5 @@ { "UniqueId": "module", - "FilePath": null, "Documentation": "", "Functions": [ { @@ -26,8 +25,7 @@ "Kind": 0 } ], - "ReturnType": null, - "Documentation": null + "ReturnType": null } ], "Documentation": null, @@ -36,7 +34,6 @@ "Functions": [], "Id": 24395611, "Name": "func", - "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 42, @@ -49,7 +46,6 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", - "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -60,7 +56,6 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", - "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -71,7 +66,6 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", - "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -82,7 +76,6 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", - "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -93,7 +86,6 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", - "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -104,7 +96,6 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", - "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -115,7 +106,6 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", - "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -126,7 +116,6 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", - "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -160,9 +149,12 @@ } ], "FileSize": 91, + "Imports": [], + "FromImports": [], + "StubImports": [], + "StubFromImports": [], "Id": -2131035837, "Name": "module", - "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/FluentAssertions/AssertionsFactory.cs b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs index fc59d467b..594897a40 100644 --- a/src/Caching/Test/FluentAssertions/AssertionsFactory.cs +++ b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Diagnostics.CodeAnalysis; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index 86ff34494..97fc44751 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -50,10 +50,8 @@ public async Task Builtins() { var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); - using (var dbModule = new PythonDbModule(model, null, Services)) { - dbModule.Construct(model); - dbModule.Should().HaveSameMembersAs(builtins); - } + var dbModule = new PythonDbModule(model, null, Services); + dbModule.Should().HaveSameMembersAs(builtins); } [TestMethod, Priority(0)] @@ -304,7 +302,7 @@ import requests // Verify this looks like a version. new Version(u.Substring(open + 1, u.IndexOf(')') - open - 1)); - await CompareRestoreAsync(model, rq); + await CompareBaselineAndRestoreAsync(model, rq); } private async Task TestModule(string name) { @@ -319,7 +317,7 @@ private async Task TestModule(string name) { var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); model.Should().NotBeNull($"Module {name} is either not installed or cannot be cached"); - await CompareRestoreAsync(model, m); + await CompareBaselineAndRestoreAsync(model, m); } } } diff --git a/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj b/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj index 11a918e64..5957991bd 100644 --- a/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj +++ b/src/Caching/Test/Microsoft.Python.Analysis.Caching.Tests.csproj @@ -32,7 +32,6 @@ - diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index 6ad1a48ae..76b9ac803 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -62,8 +62,8 @@ def methodB2(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); + var json = ToJson(model); + Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { dbModule.Construct(model); @@ -104,7 +104,7 @@ import logging var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging"); var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library); - await CompareRestoreAsync(model, logging); + await CompareBaselineAndRestoreAsync(model, logging); using (var m = CreateDbModule(model, logging.FilePath)) { var critical = m.GetMember("critical") as IPythonFunctionType; diff --git a/src/Core/Impl/Extensions/TaskExtensions.cs b/src/Core/Impl/Extensions/TaskExtensions.cs index e362c44ed..e8e3084a5 100644 --- a/src/Core/Impl/Extensions/TaskExtensions.cs +++ b/src/Core/Impl/Extensions/TaskExtensions.cs @@ -18,7 +18,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core.Testing; -using Microsoft.Python.Core.Threading; namespace Microsoft.Python.Core { public static class TaskExtensions { @@ -111,17 +110,14 @@ private static void DoNotWaitSynchronizationContextContinuation(Task task, objec /// /// Attach new to the given task. - /// This allows caller to have its own cancellation without aborting underlying work. + /// + /// this allows caller to have its own cancellation without aborting underlying work. + /// + /// if uses different cancellation token than one given + /// it will throw instead of and + /// Task will be set to faulted rather than cancelled. /// - public static Task WaitAsync(this Task task, CancellationToken cancellationToken) { - if (task.IsCompleted || !cancellationToken.CanBeCanceled) { - return task; - } - - var tcs = new TaskCompletionSource(); - tcs.RegisterForCancellation(cancellationToken).UnregisterOnCompletion(task); - task.SetCompletionResultTo(tcs); - return tcs.Task; - } + public static Task WaitAsync(this Task task, CancellationToken cancellationToken) + => task.ContinueWith(t => t.WaitAndUnwrapExceptions(), cancellationToken, TaskContinuationOptions.None, TaskScheduler.Default); } } diff --git a/src/Core/Impl/IO/PathUtils.cs b/src/Core/Impl/IO/PathUtils.cs index add25c0e0..75ecb202d 100644 --- a/src/Core/Impl/IO/PathUtils.cs +++ b/src/Core/Impl/IO/PathUtils.cs @@ -276,6 +276,10 @@ public static string GetFileName(string filePath) { /// /// true to return files within subdirectories. /// + /// + /// true to return full paths for all subdirectories. Otherwise, + /// the relative path from is returned. + /// public static IEnumerable EnumerateFiles(IFileSystem fileSystem, string root, string pattern = "*", bool recurse = true) { root = EnsureEndSeparator(root); @@ -286,16 +290,15 @@ public static IEnumerable EnumerateFiles(IFileSystem fileSystem, stri foreach (var dir in dirs) { var fullDir = Path.IsPathRooted(dir) ? dir : root + dir; - if (string.IsNullOrEmpty(fullDir)) { - continue; - } IFileInfo[] files = null; try { - files = fileSystem.GetDirectoryInfo(fullDir) - .EnumerateFileSystemInfos(pattern, SearchOption.TopDirectoryOnly) - .Where(f => !f.Attributes.HasFlag(FileAttributes.Directory)) - .OfType() - .ToArray(); + if (fileSystem.DirectoryExists(fullDir)) { + files = fileSystem.GetDirectoryInfo(fullDir) + .EnumerateFileSystemInfos(pattern, SearchOption.TopDirectoryOnly) + .Where(f => !f.Attributes.HasFlag(FileAttributes.Directory)) + .OfType() + .ToArray(); + } } catch (UnauthorizedAccessException) { } catch (IOException) { } diff --git a/src/Core/Impl/Services/ServiceManager.cs b/src/Core/Impl/Services/ServiceManager.cs index 4c8c1140f..eeaede73b 100644 --- a/src/Core/Impl/Services/ServiceManager.cs +++ b/src/Core/Impl/Services/ServiceManager.cs @@ -66,21 +66,20 @@ public IServiceManager AddService(Func factory) where T /// Service type /// Service instance or null if it doesn't exist public T GetService(Type type = null) where T : class { - //if (_disposeToken.IsDisposed) { + if (_disposeToken.IsDisposed) { // Do not throw. When editor text buffer is closed, the associated service manager // is disposed. However, some actions may still hold on the text buffer reference // and actually determine if buffer is closed by checking if editor document // is still attached as a service. - //return null; - //} + return null; + } type = type ?? typeof(T); if (!_s.TryGetValue(type, out var value)) { value = _s.FirstOrDefault(kvp => type.GetTypeInfo().IsAssignableFrom(kvp.Key)).Value; } - //return (T)CheckDisposed(value as T ?? (value as Lazy)?.Value); - return value as T ?? (value as Lazy)?.Value as T; + return (T)CheckDisposed(value as T ?? (value as Lazy)?.Value); } public void RemoveService(object service) => _s.TryRemove(service.GetType(), out var dummy); @@ -99,13 +98,13 @@ private object CheckDisposed(object service) { #region IDisposable public void Dispose() { if (_disposeToken.TryMarkDisposed()) { - //foreach (var service in _s.Values) { - // if (service is Lazy lazy && lazy.IsValueCreated) { - // (lazy.Value as IDisposable)?.Dispose(); - // } else { - // (service as IDisposable)?.Dispose(); - // } - //} + foreach (var service in _s.Values) { + if (service is Lazy lazy && lazy.IsValueCreated) { + (lazy.Value as IDisposable)?.Dispose(); + } else { + (service as IDisposable)?.Dispose(); + } + } } } #endregion diff --git a/src/Core/Impl/Threading/AsyncCountdownEvent.cs b/src/Core/Impl/Threading/AsyncCountdownEvent.cs index a4609c57f..ea3b8b761 100644 --- a/src/Core/Impl/Threading/AsyncCountdownEvent.cs +++ b/src/Core/Impl/Threading/AsyncCountdownEvent.cs @@ -21,7 +21,7 @@ namespace Microsoft.Python.Core { public class AsyncCountdownEvent { private readonly AsyncManualResetEvent _mre = new AsyncManualResetEvent(); - private long _count; + private int _count; public AsyncCountdownEvent(int initialCount) { if (initialCount < 0) { @@ -34,7 +34,7 @@ public AsyncCountdownEvent(int initialCount) { } } - public long Count => Interlocked.Read(ref _count); + public Task WaitAsync() => _mre.WaitAsync(); public Task WaitAsync(CancellationToken cancellationToken) => _mre.WaitAsync(cancellationToken); diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 4d1e3cc5c..b9e425d74 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -160,7 +160,7 @@ public async Task InitializedAsync(InitializedParams @params, CancellationToken interpreterPath: interpeterPath, version: version ); - _services.AddService(new ModuleDatabase(_services)); + //_services.AddService(new ModuleDatabase(_services)); var typeshedPath = initializationOptions?.typeStubSearchPaths.FirstOrDefault(); userConfiguredPaths = userConfiguredPaths ?? initializationOptions?.searchPaths; diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index 03a8be120..f5da88912 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -58,13 +58,12 @@ public Task ParseAsync(string path, CancellationToken cancellationTok private async Task Parse(string path, CancellationToken parseCt) { await _semaphore.WaitAsync(parseCt); - PythonAst ast = null; + PythonAst ast; try { - await using var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read); - var parser = Parser.CreateParser(stream, _version); - ast = parser.ParseFile(new Uri(path)); - } catch(Exception ex) when (!ex.IsCriticalException()) { - return null; + using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { + var parser = Parser.CreateParser(stream, _version); + ast = parser.ParseFile(new Uri(path)); + } } finally { _semaphore.Release(); } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index acf8bcca9..ae9593025 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -154,12 +154,10 @@ private async Task> IndexAsync(IDocument doc, private async Task> ParseAsync(CancellationToken cancellationToken) { try { var ast = await _indexParser.ParseAsync(_path, cancellationToken); - if (ast != null) { - cancellationToken.ThrowIfCancellationRequested(); - var walker = new SymbolIndexWalker(ast, _library, cancellationToken); - ast.Walk(walker); - return walker.Symbols; - } + cancellationToken.ThrowIfCancellationRequested(); + var walker = new SymbolIndexWalker(ast, _library, cancellationToken); + ast.Walk(walker); + return walker.Symbols; } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); } diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 73a4c2fe0..0a286f30f 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -224,17 +224,21 @@ private ImmutableArray GetUserConfiguredPaths(JToken pythonSection) { private const string DefaultCachingLevel = "None"; private AnalysisCachingLevel GetAnalysisCachingLevel(JToken analysisKey) { - var s = GetSetting(analysisKey, "cachingLevel", DefaultCachingLevel); - - if (string.IsNullOrWhiteSpace(s) || s.EqualsIgnoreCase("Default")) { - s = DefaultCachingLevel; - } - - if (s.EqualsIgnoreCase("System")) { - return AnalysisCachingLevel.System; - } - - return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; + // TODO: Remove this one caching is working at any level again. + // https://github.com/microsoft/python-language-server/issues/1758 + return AnalysisCachingLevel.None; + + // var s = GetSetting(analysisKey, "cachingLevel", DefaultCachingLevel); + // + // if (string.IsNullOrWhiteSpace(s) || s.EqualsIgnoreCase("Default")) { + // s = DefaultCachingLevel; + // } + // + // if (s.EqualsIgnoreCase("System")) { + // return AnalysisCachingLevel.System; + // } + // + // return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; } } } diff --git a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs index 93e8431c0..811d7bf22 100644 --- a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs +++ b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs @@ -116,7 +116,7 @@ private void MonitorParentProcess(InitializeParams p) { Debug.Assert(parentProcess != null, "Parent process does not exist"); if (parentProcess != null) { - parentProcess.Exited += (s, e) => _sessionTokenSource.Cancel(); + parentProcess.Exited += (s, e) => TerminateProcess(); } } @@ -125,13 +125,18 @@ private void MonitorParentProcess(InitializeParams p) { while (!_sessionTokenSource.IsCancellationRequested) { await Task.Delay(2000); if (parentProcess.HasExited) { - _sessionTokenSource.Cancel(); + TerminateProcess(); } } }).DoNotWait(); } } + private void TerminateProcess() { + _sessionTokenSource?.Cancel(); + Environment.Exit(0); + } + private void RegisterServices(InitializeParams initParams) { // we need to register cache service first. // optimization service consumes the cache info. diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index b16d6dc08..3df9d7b1d 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -311,7 +311,6 @@ public async Task ExtensionCommand(JToken token, CancellationToken cancellationT public async Task ClearAnalysisCache(CancellationToken cancellationToken) { using (_requestTimer.Time("python/clearAnalysisCache")) using (await _prioritizer.ConfigurationPriorityAsync(cancellationToken)) { - // Debug.Assert(_initialized); _server.ClearAnalysisCache(); } } diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index 82cc9a0e2..e129781d0 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -35,7 +35,6 @@ - diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index fdd3f3a28..3de1b1424 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -602,15 +602,13 @@ from os import path as os_path reference = ds.FindDefinition(analysis, new SourceLocation(4, 12), out _); reference.Should().NotBeNull(); - var osPyPath = reference.uri.AbsolutePath; - line = File.ReadAllLines(osPyPath)[reference.range.start.line]; + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; line.Should().EndWith("as path"); line.Substring(reference.range.start.character).Should().Be("path"); reference = ds.FindDefinition(analysis, new SourceLocation(5, 12), out _); reference.Should().NotBeNull(); - reference.uri.AbsolutePath.Should().Be(osPyPath); - line = File.ReadAllLines(osPyPath)[reference.range.start.line]; + line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; line.Should().EndWith("as path"); line.Substring(reference.range.start.character).Should().Be("path"); } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index ef60ad410..b114e05ff 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -475,129 +475,6 @@ from module3 import A3 comps.Should().HaveLabels("M1"); } - [TestMethod, Priority(0)] - public async Task LoopImports_Variables1() { - const string module1Code = @" -class A1: - def M1(self): return 0; pass - -from module2 import y3 -x = y3.M3() -"; - const string module2Code = @" -from module1 import A1 -y1 = A1() -from module3 import A3 -y3 = A3() -"; - const string module3Code = @" -class A3: - def M3(self): return '0'; pass - -from module2 import y1 -z = y1.M1() -"; - - const string appCode = @" -from module1 import x -from module3 import z - -x. -z."; - var module1Uri = TestData.GetTestSpecificUri("module1.py"); - var module2Uri = TestData.GetTestSpecificUri("module2.py"); - var module3Uri = TestData.GetTestSpecificUri("module3.py"); - var appUri = TestData.GetTestSpecificUri("app.py"); - - var root = Path.GetDirectoryName(appUri.AbsolutePath); - await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); - var rdt = Services.GetService(); - var analyzer = Services.GetService(); - - rdt.OpenDocument(module3Uri, module3Code); - rdt.OpenDocument(module2Uri, module2Code); - rdt.OpenDocument(module1Uri, module1Code); - - var app = rdt.OpenDocument(appUri, appCode); - await analyzer.WaitForCompleteAnalysisAsync(); - var analysis = await app.GetAnalysisAsync(-1); - - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); - var comps = cs.GetCompletions(analysis, new SourceLocation(5, 3)); - comps.Should().HaveLabels("capitalize"); - - comps = cs.GetCompletions(analysis, new SourceLocation(6, 3)); - comps.Should().HaveLabels("bit_length"); - } - - [TestMethod, Priority(0)] - public async Task LoopImports_Variables2() { - const string module1Code = @" -from module3 import A3 - -class A1: - def M1(self) -> A3: pass - -from module2 import y3 -x = y3.M3() -"; - const string module2Code = @" -from module1 import A1 -y1 = A1() -from module3 import A3 -y3 = A3() -"; - const string module3Code = @" -from module1 import A1 - -class A3: - def M3(self) -> A1: pass - -from module2 import y1 -z = y1.M1() -"; - const string appCode = @" -from module1 import x -from module3 import z - -x. -z. - -x.M1(). -z.M3(). -"; - var module1Uri = TestData.GetTestSpecificUri("module1.py"); - var module2Uri = TestData.GetTestSpecificUri("module2.py"); - var module3Uri = TestData.GetTestSpecificUri("module3.py"); - var appUri = TestData.GetTestSpecificUri("app.py"); - - var root = Path.GetDirectoryName(appUri.AbsolutePath); - await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); - var rdt = Services.GetService(); - var analyzer = Services.GetService(); - - rdt.OpenDocument(module3Uri, module3Code); - rdt.OpenDocument(module2Uri, module2Code); - rdt.OpenDocument(module1Uri, module1Code); - - var app = rdt.OpenDocument(appUri, appCode); - await analyzer.WaitForCompleteAnalysisAsync(); - var analysis = await app.GetAnalysisAsync(-1); - - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); - var comps = cs.GetCompletions(analysis, new SourceLocation(5, 3)); - comps.Should().HaveLabels("M1"); - - comps = cs.GetCompletions(analysis, new SourceLocation(6, 3)); - comps.Should().HaveLabels("M3"); - - comps = cs.GetCompletions(analysis, new SourceLocation(8, 8)); - comps.Should().HaveLabels("M3"); - - comps = cs.GetCompletions(analysis, new SourceLocation(9, 8)); - comps.Should().HaveLabels("M1"); - } - [TestMethod, Priority(0)] public async Task TypingModule() { var analysis = await GetAnalysisAsync(@"from typing import "); @@ -809,12 +686,13 @@ def bar(self): pass " + allCode; - const string appCode = @" + var appCode = @" from module1 import * A(). B(). "; + var module1Uri = TestData.GetTestSpecificUri("module1.py"); var appUri = TestData.GetTestSpecificUri("app.py"); diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 693bd25ff..32e66ec26 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -74,6 +74,18 @@ private IReadOnlyList GetIndexSymbols(PythonAst ast) { } + [TestMethod, Priority(0)] + [ExpectedException(typeof(FileNotFoundException))] + public async Task ParseFileThatStopsExisting() { + const string testFilePath = "C:/bla.py"; + _fileSystem.FileExists(testFilePath).Returns(true); + SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); + + using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { + await indexParser.ParseAsync(testFilePath); + } + } + [TestMethod, Priority(0)] public void CancelParsingAsync() { const string testFilePath = "C:/bla.py"; diff --git a/src/LanguageServer/Test/LanguageServerTestBase.cs b/src/LanguageServer/Test/LanguageServerTestBase.cs index 7ef8e7e93..8537b2d81 100644 --- a/src/LanguageServer/Test/LanguageServerTestBase.cs +++ b/src/LanguageServer/Test/LanguageServerTestBase.cs @@ -26,6 +26,7 @@ public abstract class LanguageServerTestBase : AnalysisTestBase { protected static readonly ServerSettings ServerSettings = new ServerSettings(); protected override IDiagnosticsService GetDiagnosticsService(IServiceContainer s) => new DiagnosticsService(s); + protected IDiagnosticsService GetDiagnosticsService() { var ds = Services.GetService(); ds.PublishingDelay = 0; diff --git a/src/LanguageServer/Test/MissingImportCodeActionTests.cs b/src/LanguageServer/Test/MissingImportCodeActionTests.cs index ca30580a9..69759b049 100644 --- a/src/LanguageServer/Test/MissingImportCodeActionTests.cs +++ b/src/LanguageServer/Test/MissingImportCodeActionTests.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis; @@ -31,6 +32,7 @@ using Microsoft.Python.LanguageServer.Sources; using Microsoft.Python.LanguageServer.Tests.FluentAssertions; using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; using Microsoft.Python.UnitTests.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -39,17 +41,16 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class MissingImportCodeActionTests : LanguageServerTestBase { public TestContext TestContext { get; set; } + public CancellationToken CancellationToken => TestContext.CancellationTokenSource.Token; [TestInitialize] - public void TestInitialize() { - TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); - AnalysisTimeout = TimeSpan.FromMinutes(3); - } + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task Missing() { MarkupUtils.GetSpan(@"[|missingModule|]", out var code, out var span); @@ -57,11 +58,11 @@ public async Task Missing() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); codeActions.Should().BeEmpty(); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task TopModule() { const string markup = @"{|insertionSpan:|}{|diagnostic:ntpath|}"; @@ -72,7 +73,7 @@ public async Task TopModule() { TestCodeAction(analysis.Document.Uri, codeAction, title: "import ntpath", insertionSpan, newText); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task TopModuleFromFunctionInsertTop() { const string markup = @"{|insertionSpan:|}def TestMethod(): {|diagnostic:ntpath|}"; @@ -86,7 +87,7 @@ public async Task TopModuleFromFunctionInsertTop() { TestCodeAction(analysis.Document.Uri, codeAction, title: "import ntpath", insertionSpan, newText); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task TopModuleLocally() { const string markup = @"def TestMethod(): {|insertionSpan:|} {|diagnostic:ntpath|}"; @@ -100,7 +101,7 @@ public async Task TopModuleLocally() { TestCodeAction(analysis.Document.Uri, codeAction, title: string.Format(Resources.ImportLocally, "import ntpath"), insertionSpan, newText); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SubModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:util|}", @@ -108,7 +109,7 @@ await TestCodeActionAsync( newText: "from ctypes import util" + Environment.NewLine + Environment.NewLine); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SubModuleUpdate() { await TestCodeActionAsync( @"{|insertionSpan:from ctypes import util|} @@ -117,7 +118,7 @@ await TestCodeActionAsync( newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task SubModuleUpdateLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -127,7 +128,7 @@ await TestCodeActionAsync( newText: "from ctypes import test, util"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SubModuleFromFunctionInsertTop() { await TestCodeActionAsync( @"{|insertionSpan:|}def TestMethod(): @@ -137,7 +138,7 @@ from ctypes import util newText: "from ctypes import test" + Environment.NewLine + Environment.NewLine); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task AfterExistingImport() { await TestCodeActionAsync( @"from os import path @@ -147,7 +148,7 @@ await TestCodeActionAsync( newText: "from ctypes import util" + Environment.NewLine); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task ReplaceExistingImport() { await TestCodeActionAsync( @"from os import path @@ -159,7 +160,7 @@ import socket newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task AfterExistingImportLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -170,7 +171,7 @@ from os import path newText: " from ctypes import util" + Environment.NewLine); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task ReplaceExistingImportLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -183,7 +184,7 @@ import socket newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore] + [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] public async Task CodeActionOrdering() { MarkupUtils.GetSpan(@"def TestMethod(): [|test|]", out var code, out var span); @@ -192,7 +193,7 @@ public async Task CodeActionOrdering() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); var list = codeActions.Select(c => c.title).ToList(); var zipList = Enumerable.Range(0, list.Count).Zip(list); @@ -204,7 +205,7 @@ public async Task CodeActionOrdering() { maxIndexOfTopAddImports.Should().BeLessThan(minIndexOfLocalAddImports); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task PreserveComment() { await TestCodeActionAsync( @"{|insertionSpan:from os import pathconf|} # test @@ -214,7 +215,7 @@ await TestCodeActionAsync( newText: "from os import path, pathconf"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task MemberSymbol() { await TestCodeActionAsync( @"from os import path @@ -224,7 +225,7 @@ await TestCodeActionAsync( newText: "from socket import socket" + Environment.NewLine); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task NoMemberSymbol() { var markup = @"{|insertionSpan:|}{|diagnostic:socket|}"; @@ -238,7 +239,7 @@ public async Task NoMemberSymbol() { TestCodeAction(analysis.Document.Uri, codeAction, title, insertionSpan, newText); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SymbolOrdering() { var markup = @"from os import path {|insertionSpan:|} @@ -252,10 +253,10 @@ public async Task SymbolOrdering() { var maxIndexOfPublicSymbol = zipList.Where(t => !t.Second.StartsWith("from _")).Max(t => t.First); var minIndexOfPrivateSymbol = zipList.Where(t => t.Second.StartsWith("from _")).Min(t => t.First); - maxIndexOfPublicSymbol.Should().BeLessThan(minIndexOfPrivateSymbol); + minIndexOfPrivateSymbol.Should().BeLessThan(maxIndexOfPublicSymbol); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS*2)] public async Task SymbolOrdering2() { var markup = @"from os import path {|insertionSpan:|} @@ -274,7 +275,7 @@ public async Task SymbolOrdering2() { importedMemberIndex.Should().BeLessThan(restIndex); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SymbolOrdering3() { var markup = @"{|insertionSpan:|}{|diagnostic:pd|}"; @@ -289,7 +290,7 @@ public async Task SymbolOrdering3() { // calculate actions var diagnosticSpan = spans["diagnostic"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, diagnosticSpan, MissingImportCodeActionProvider.Instance.FixableDiagnostics); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); var list = codeActions.Select(c => c.title).ToList(); var zipList = Enumerable.Range(0, list.Count).Zip(list); @@ -300,7 +301,8 @@ public async Task SymbolOrdering3() { pandasIndex.Should().BeLessThan(pdIndex); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [Ignore("loops")] public async Task ModuleNotReachableFromUserDocument() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:path|}", @@ -309,7 +311,7 @@ await TestCodeActionAsync( enableIndexManager: true); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestAbbreviationForKnownModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pandas|}", @@ -319,7 +321,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestAbbreviationForKnownModule2() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pyplot|}", @@ -329,7 +331,7 @@ await TestCodeActionAsync( relativePaths: @"matplotlib\pyplot.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestAbbreviationForKnownModule3() { var markup = @" {|insertionSpan:from matplotlib import test|} @@ -343,7 +345,7 @@ await TestCodeActionAsync( relativePaths: new string[] { @"matplotlib\pyplot.py", @"matplotlib\test.py" }); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestReverseAbbreviationForKnownModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pd|}", @@ -353,7 +355,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestReverseAbbreviationForKnownModule2() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:plt|}", @@ -363,7 +365,7 @@ await TestCodeActionAsync( relativePaths: @"matplotlib\pyplot.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task SuggestReverseAbbreviationForKnownModule3() { var markup = @" {|insertionSpan:from matplotlib import test|} @@ -377,7 +379,7 @@ await TestCodeActionAsync( relativePaths: new string[] { @"matplotlib\pyplot.py", @"matplotlib\test.py" }); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task AbbreviationConflict() { var markup = @"{|insertionSpan:|}pd = 1 @@ -391,7 +393,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task AbbreviationConflict2() { var markup = @"{|insertionSpan:|}{|diagnostic:pandas|} @@ -406,7 +408,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task ContextBasedSuggestion() { var markup = @"from os import path @@ -425,7 +427,7 @@ public async Task ContextBasedSuggestion() { TestCodeAction(analysis.Document.Uri, codeAction, title, insertionSpan, newText); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task ValidToBeUsedInImport() { await TestCodeActionAsync( @"from os import path @@ -435,7 +437,7 @@ await TestCodeActionAsync( newText: "from os.path import join" + Environment.NewLine); } - [TestMethod, Priority(0)] + [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] public async Task Disabled() { var markup = @"from os import path [|socket|]()"; @@ -446,11 +448,11 @@ public async Task Disabled() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); codeActions.Should().NotBeEmpty(); var emptyActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync( - analysis, new CodeActionSettings(null, new Dictionary() { { "addimports", false } }), diagnostics, TestCancellationToken); + analysis, new CodeActionSettings(null, new Dictionary() { { "addimports", false } }), diagnostics, CancellationToken); emptyActions.Should().BeEmpty(); } @@ -487,7 +489,7 @@ private async Task TestCodeActionAsync(string markup, string title, string newTe var insertionSpan = spans["insertionSpan"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, spans["diagnostic"].First().ToSourceSpan(analysis.Ast), codes); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); return (analysis, codeActions.ToArray(), insertionSpan); } @@ -521,7 +523,7 @@ private async Task TestCodeActionAsync(string markup, string title, string newTe // calculate actions var diagnosticSpan = spans["diagnostic"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, diagnosticSpan, MissingImportCodeActionProvider.Instance.FixableDiagnostics); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); // verify results var codeAction = codeActions.Single(c => c.title == title); diff --git a/src/PLS.sln b/src/PLS.sln index 2bceb6b86..ab969a558 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -27,10 +27,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.T EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Tests", "LanguageServer\Test\Microsoft.Python.LanguageServer.Tests.csproj", "{3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Caching", "Caching\Impl\Microsoft.Python.Analysis.Caching.csproj", "{42BD3C80-3E57-4847-8142-84F6B682EA8D}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Caching.Tests", "Caching\Test\Microsoft.Python.Analysis.Caching.Tests.csproj", "{40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -77,14 +73,6 @@ Global {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Debug|Any CPU.Build.0 = Debug|Any CPU {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.ActiveCfg = Release|Any CPU {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.Build.0 = Release|Any CPU - {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {42BD3C80-3E57-4847-8142-84F6B682EA8D}.Release|Any CPU.Build.0 = Release|Any CPU - {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -100,8 +88,6 @@ Global {2C8DE250-41F4-4FC5-A661-76E2A4172891} = {C465393D-145E-4695-A7DB-AF55951BD533} {D8D85896-5DB0-4FA6-B744-910A272C39F9} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} - {42BD3C80-3E57-4847-8142-84F6B682EA8D} = {C465393D-145E-4695-A7DB-AF55951BD533} - {40CD3A74-B0B6-4A37-AE65-5B203C38D0E2} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ABC12ED7-0EC8-4219-8A14-A058F7942D92} diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 58c99e8c4..5ebeee6e7 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -49,14 +49,12 @@ public PythonAst(IEnumerable existingAst) { locs.AddRange(a.NewLineLocations.Select(ll => new NewLineLocation(ll.EndIndex + offset, ll.Kind))); offset = locs.LastOrDefault().EndIndex; } - NewLineLocations = locs.ToArray(); offset = 0; foreach (var a in asts) { comments.AddRange(a.CommentLocations.Select(cl => new SourceLocation(cl.Line + offset, cl.Column))); offset += a.NewLineLocations.Length + 1; } - CommentLocations = comments.ToArray(); } @@ -76,13 +74,12 @@ public PythonAst(IEnumerable existingAst) { /// public bool HasVerbatim { get; internal set; } - public override IEnumerable GetChildNodes() => new[] {_body}; + public override IEnumerable GetChildNodes() => new[] { _body }; public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { _body.Walk(walker); } - walker.PostWalk(this); } @@ -90,16 +87,15 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken if (await walker.WalkAsync(this, cancellationToken)) { await _body.WalkAsync(walker, cancellationToken); } - await walker.PostWalkAsync(this, cancellationToken); } public override Statement Body => _body; public PythonLanguageVersion LanguageVersion { get; } - public void ReduceToImports() { + public void Reduce(Func filter) { lock (_lock) { - (Body as SuiteStatement)?.ReduceToImports(); + (Body as SuiteStatement)?.FilterStatements(filter); _attributes?.Clear(); Variables?.Clear(); CommentLocations = Array.Empty(); @@ -125,7 +121,6 @@ public void SetAttribute(Node node, object key, object value) { if (!_attributes.TryGetValue(node, out var nodeAttrs)) { nodeAttrs = _attributes[node] = new Dictionary(); } - nodeAttrs[key] = value; } } @@ -146,10 +141,8 @@ internal void SetAttributes(Dictionary> attribu } #region ILocationConverter - public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); - #endregion internal int GetLineEndFromPosition(int index) { @@ -157,7 +150,6 @@ internal int GetLineEndFromPosition(int index) { if (loc.Line >= NewLineLocations.Length) { return index; } - var res = NewLineLocations[loc.Line - 1]; switch (res.Kind) { case NewLineKind.LineFeed: @@ -172,7 +164,8 @@ internal int GetLineEndFromPosition(int index) { internal override bool ExposesLocalVariable(PythonVariable variable) => true; - internal override void FinishBind(PythonNameBinder binder) { } + internal override void FinishBind(PythonNameBinder binder) { + } internal override PythonVariable BindReference(PythonNameBinder binder, string name) => EnsureVariable(name); @@ -193,7 +186,6 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow return true; } } - variable = null; return false; } @@ -205,7 +197,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow /// for variables explicitly declared global by the user, and names accessed /// but not defined in the lexical scope. /// - internal PythonVariable /*!*/ EnsureGlobalVariable(string name) { + internal PythonVariable/*!*/ EnsureGlobalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Global); } @@ -214,7 +206,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow } - internal PythonVariable /*!*/ EnsureNonlocalVariable(string name) { + internal PythonVariable/*!*/ EnsureNonlocalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Nonlocal); } diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index 25fd90d72..920a1d8b6 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -14,7 +14,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -31,8 +33,8 @@ public SuiteStatement(Statement[] statements) { public IList Statements => _statements; public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); - public void ReduceToImports() - => _statements = new FilteredWalker(this).Statements.ToArray(); + public void FilterStatements(Func filter) + => _statements = _statements.Where(filter).ToArray(); public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { @@ -162,22 +164,5 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) { _statements[0].SetLeadingWhiteSpace(ast, whiteSpace); } } - - private sealed class FilteredWalker : PythonWalker { - public FilteredWalker(Node n) { - n.Walk(this); - } - - public List Statements { get; } = new List(); - - public override bool Walk(ImportStatement s) { - Statements.Add(s); - return false; - } - public override bool Walk(FromImportStatement s) { - Statements.Add(s); - return false; - } - } } } diff --git a/src/UnitTests/Core/Impl/Baseline.cs b/src/UnitTests/Core/Impl/Baseline.cs index 0c829a12e..74e87d235 100644 --- a/src/UnitTests/Core/Impl/Baseline.cs +++ b/src/UnitTests/Core/Impl/Baseline.cs @@ -93,6 +93,7 @@ public static int CompareLines(string expected, string actual, out string expect if (expectedLine == null && actualLine == null) { expectedLine = string.Empty; actualLine = string.Empty; + return 0; } diff --git a/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs b/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs index 3a34a8ba0..7f94fb7e5 100644 --- a/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs +++ b/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs @@ -28,22 +28,16 @@ namespace TestUtilities { public class TestEnvironmentImpl { private static readonly FieldInfo _stackTraceStringField = typeof(Exception).GetField("_stackTraceString", BindingFlags.Instance | BindingFlags.NonPublic); private static readonly FieldInfo _showDialogField = typeof(Debug).GetField("s_ShowDialog", BindingFlags.Static | BindingFlags.NonPublic); - private static readonly FieldInfo _debugProviderField = typeof(Debug).GetField("s_provider", BindingFlags.Static | BindingFlags.NonPublic); protected internal static TestEnvironmentImpl Instance { get; protected set; } protected TestEnvironmentImpl() { - TryOverrideDebugFail(); + TryOverrideShowDialog(); } - private static void TryOverrideDebugFail() { + private static void TryOverrideShowDialog() { if (_showDialogField != null) { _showDialogField.SetValue(null, new Action(ThrowAssertException)); - } else if (_debugProviderField != null) { - var failCoreField = _debugProviderField.FieldType.GetField("s_FailCore", BindingFlags.Static | BindingFlags.NonPublic); - if (failCoreField != null) { - failCoreField.SetValue(null, new Action(ThrowAssertException)); - } } }