From 6a9843e041148566425d7542c6b9ebdd8d2b1940 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 10:57:49 -0700 Subject: [PATCH 01/19] Add submodules to module members --- .../Impl/Extensions/PythonModuleExtensions.cs | 38 +++++++++++++++++++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 13 ++++++- .../Impl/DependencyResolution/AstUtilities.cs | 3 +- .../Impl/Completion/ImportCompletion.cs | 30 +++------------ 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index caa2b5b34..7571031b0 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -13,8 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; @@ -70,5 +74,39 @@ internal static string GetComment(this IPythonModule module, int lineNum) { internal static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); internal static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); + + public static IEnumerable GetSubmoduleNames(this IPythonModule module) { + var searchResult = module.Interpreter.ModuleResolution.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(module.Name, 1), 0, true); + if (searchResult is IImportChildrenSource children) { + foreach (var name in children.GetChildrenNames()) { + if (children.TryGetChildImport(name, out var imports)) { + switch (imports) { + case ImplicitPackageImport packageImport: + yield return packageImport.Name; + break; + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): + yield return moduleImport.Name; + break; + } + } + } + } + } + + public static IPythonModule GetSubmodule(this IPythonModule module, string subModuleName) { + var mres = module.Interpreter.ModuleResolution; + var searchResult = mres.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(module.Name, 1), 0, true); + if (searchResult is IImportChildrenSource children) { + if (children.TryGetChildImport(subModuleName, out var imports)) { + switch (imports) { + case ImplicitPackageImport packageImport: + return mres.GetImportedModule(packageImport.FullName); + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): + return mres.GetImportedModule(moduleImport.FullName); + } + } + } + return null; + } } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index aae9465c2..bc4fe5157 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -157,7 +158,9 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; + public virtual IMember GetMember(string name) + => GlobalScope.Variables[name]?.Value ?? this.GetSubmodule(name); + public virtual IEnumerable GetMemberNames() { // drop imported modules and typing. return GlobalScope.Variables @@ -166,16 +169,20 @@ public virtual IEnumerable GetMemberNames() { if (v.Value is IPythonInstance) { return true; } + var valueType = v.Value?.GetPythonType(); if (valueType is PythonModule) { return false; // Do not re-export modules. } + if (valueType is IPythonFunctionType f && f.IsLambda()) { return false; } + if (this is TypingModule) { return true; // Let typing module behave normally. } + // Do not re-export types from typing. However, do export variables // assigned with types from typing. Example: // from typing import Any # do NOT export Any @@ -183,9 +190,11 @@ public virtual IEnumerable GetMemberNames() { if (valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) { return false; } + return true; }) - .Select(v => v.Name); + .Select(v => v.Name) + .Concat(this.GetSubmoduleNames()); } #endregion diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs index 57c93cc1d..8c7ee1457 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs @@ -13,9 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; -using System.Linq; using Microsoft.Python.Parsing.Ast; +using System.Collections.Generic; namespace Microsoft.Python.Analysis.Core.DependencyResolution { public static class AstUtilities { diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 9fb15e009..eec624ee4 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -15,11 +15,10 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; +using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; @@ -41,7 +40,7 @@ public static CompletionResult TryGetCompletions(ImportStatement import, Complet if (name != null && context.Position >= name.StartIndex) { if (context.Position > name.EndIndex && name.EndIndex > name.StartIndex) { var applicableSpan = context.GetApplicableSpanFromLastToken(import); - return new CompletionResult(new []{ CompletionItemSource.AsKeyword }, applicableSpan); + return new CompletionResult(new[] { CompletionItemSource.AsKeyword }, applicableSpan); } if (name.Names.Count == 0 || name.Names[0].EndIndex >= context.Position) { @@ -60,7 +59,7 @@ public static CompletionResult TryGetCompletions(ImportStatement import, Complet public static CompletionResult GetCompletionsInFromImport(FromImportStatement fromImport, CompletionContext context) { // No more completions after '*', ever! - if (fromImport.Names != null && fromImport.Names.Any(n => n?.Name == "*" && context.Position > n.EndIndex)) { + if (fromImport.Names.Any(n => n?.Name == "*" && context.Position > n.EndIndex)) { return CompletionResult.Empty; } @@ -160,7 +159,7 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im default: return CompletionResult.Empty; } - + var completions = new List(); if (prependStar) { completions.Add(CompletionItemSource.Star); @@ -168,25 +167,8 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im if (module != null) { completions.AddRange(module.GetMemberNames() - .Where(n => !string.IsNullOrEmpty(n)) - .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); - } - - if (importSearchResult is IImportChildrenSource children) { - foreach (var childName in children.GetChildrenNames()) { - if (!children.TryGetChildImport(childName, out var imports)) { - continue; - } - - switch (imports) { - case ImplicitPackageImport packageImport: - completions.Add(CompletionItemSource.CreateCompletionItem(packageImport.Name, CompletionItemKind.Module)); - break; - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(document.FilePath): - completions.Add(CompletionItemSource.CreateCompletionItem(moduleImport.Name, CompletionItemKind.Module)); - break; - } - } + .Where(n => !string.IsNullOrEmpty(n)) + .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); } return new CompletionResult(completions, applicableSpan); From e9d37c78e2c823bfbccc13e5ff5d4b0637981cfa Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 12:36:03 -0700 Subject: [PATCH 02/19] Add test --- src/Analysis/Ast/Test/ImportTests.cs | 17 +++++++++++++++++ .../Impl/Completion/ImportCompletion.cs | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 3e28e8a1b..62be46260 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -13,10 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; @@ -234,5 +237,19 @@ def exit(): var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Int); } + + [TestMethod, Priority(0)] + public async Task ModuleMembers() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "import m1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var doc = rdt.OpenDocument(appUri, "import package"); + + var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1"); + } } } diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index eec624ee4..72885ce00 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Text; From fa1bc062a0e0d9397b788fa129102909efa3139d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 13:09:35 -0700 Subject: [PATCH 03/19] Test updates --- .../Impl/Completion/ImportCompletion.cs | 18 ++++++++++++++++ src/LanguageServer/Test/CompletionTests.cs | 21 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 72885ce00..af988f489 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -18,6 +18,7 @@ using System.Linq; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; @@ -168,6 +169,23 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im completions.AddRange(module.GetMemberNames() .Where(n => !string.IsNullOrEmpty(n)) .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); + } else { + if (importSearchResult is IImportChildrenSource children) { + foreach (var childName in children.GetChildrenNames()) { + if (!children.TryGetChildImport(childName, out var imports)) { + continue; + } + + switch (imports) { + case ImplicitPackageImport packageImport: + completions.Add(CompletionItemSource.CreateCompletionItem(packageImport.Name, CompletionItemKind.Module)); + break; + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(document.FilePath): + completions.Add(CompletionItemSource.CreateCompletionItem(moduleImport.Name, CompletionItemKind.Module)); + break; + } + } + } } return new CompletionResult(completions, applicableSpan); diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index 625eba996..787fe7274 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -16,6 +16,7 @@ using System; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis; @@ -1012,7 +1013,7 @@ public async Task FromDotInRootWithInitPy() { var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); - result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); + result.Should().OnlyHaveLabels("module1", "__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); } [TestMethod, Priority(0)] @@ -1084,6 +1085,24 @@ public async Task FromDotInImplicitPackage() { result.Should().OnlyHaveLabels("module2", "sub_package"); } + [TestMethod, Priority(0)] + public async Task SubmoduleMember() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "import m1\nx = 1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m2", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var doc = rdt.OpenDocument(appUri, "import package\npackage."); + + var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = cs.GetCompletions(analysis, new SourceLocation(2, 9)); + result.Should().HaveLabels("m1", "m2", "x"); + } + [DataRow(false)] [DataRow(true)] [DataTestMethod, Priority(0)] From 30303866b495aa971e20a3d0900a7fb96c6e5620 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 13:14:21 -0700 Subject: [PATCH 04/19] Wait for complete analysis --- src/LanguageServer/Test/CompletionTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index 787fe7274..f39b825c9 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1096,6 +1096,7 @@ public async Task SubmoduleMember() { var rdt = Services.GetService(); var doc = rdt.OpenDocument(appUri, "import package\npackage."); + await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); From eb2e8f91c5161a4376988aedfc24564c4c524b8d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 15:31:35 -0700 Subject: [PATCH 05/19] Different way of filtering --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 27 ++++++++--------- .../FluentAssertions/VariableAssertions.cs | 3 +- src/Analysis/Ast/Test/ImportTests.cs | 6 ++-- .../Impl/Completion/ImportCompletion.cs | 30 +++++++++---------- src/LanguageServer/Test/CompletionTests.cs | 4 +-- 5 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index bc4fe5157..8e52f8669 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -158,10 +158,12 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) - => GlobalScope.Variables[name]?.Value ?? this.GetSubmodule(name); + public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; public virtual IEnumerable GetMemberNames() { + var fs = Services.GetService(); + var thisModuleDirectory = Path.GetDirectoryName(FilePath); + // drop imported modules and typing. return GlobalScope.Variables .Where(v => { @@ -171,12 +173,12 @@ public virtual IEnumerable GetMemberNames() { } var valueType = v.Value?.GetPythonType(); - if (valueType is PythonModule) { - return false; // Do not re-export modules. - } - - if (valueType is IPythonFunctionType f && f.IsLambda()) { - return false; + switch (valueType) { + case IPythonModule m: + // Do not re-export modules except submodules. + return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); + case IPythonFunctionType f when f.IsLambda(): + return false; } if (this is TypingModule) { @@ -187,14 +189,9 @@ public virtual IEnumerable GetMemberNames() { // assigned with types from typing. Example: // from typing import Any # do NOT export Any // x = Union[int, str] # DO export x - if (valueType?.DeclaringModule is TypingModule && v.Name == valueType.Name) { - return false; - } - - return true; + return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; }) - .Select(v => v.Name) - .Concat(this.GetSubmoduleNames()); + .Select(v => v.Name); } #endregion diff --git a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs index 6f0aac0c1..6758d2a09 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs @@ -65,8 +65,7 @@ public AndWhichConstraint HaveMember(string name, s public AndWhichConstraint NotHaveMember(string name, string because = "", params object[] reasonArgs) { NotBeNull(because, reasonArgs); - var m = Value.GetPythonType().GetMember(name); - m.GetPythonType().IsUnknown().Should().BeTrue(); + Value.GetPythonType().GetMemberNames().Should().NotContain(name); return new AndWhichConstraint(this, Subject); } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 62be46260..0ce949174 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -18,6 +18,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; @@ -241,15 +242,16 @@ def exit(): [TestMethod, Priority(0)] public async Task ModuleMembers() { var appUri = TestData.GetTestSpecificUri("app.py"); - await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "import m1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "from . import m1\nimport sys"); await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); await CreateServicesAsync(PythonVersions.LatestAvailable3X); var rdt = Services.GetService(); var doc = rdt.OpenDocument(appUri, "import package"); + await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1"); + analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1").And.NotHaveMember("sys"); } } } diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index af988f489..e02c5264d 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -169,21 +169,21 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im completions.AddRange(module.GetMemberNames() .Where(n => !string.IsNullOrEmpty(n)) .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); - } else { - if (importSearchResult is IImportChildrenSource children) { - foreach (var childName in children.GetChildrenNames()) { - if (!children.TryGetChildImport(childName, out var imports)) { - continue; - } - - switch (imports) { - case ImplicitPackageImport packageImport: - completions.Add(CompletionItemSource.CreateCompletionItem(packageImport.Name, CompletionItemKind.Module)); - break; - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(document.FilePath): - completions.Add(CompletionItemSource.CreateCompletionItem(moduleImport.Name, CompletionItemKind.Module)); - break; - } + } + + if (importSearchResult is IImportChildrenSource children) { + foreach (var childName in children.GetChildrenNames()) { + if (!children.TryGetChildImport(childName, out var imports)) { + continue; + } + + switch (imports) { + case ImplicitPackageImport packageImport: + completions.Add(CompletionItemSource.CreateCompletionItem(packageImport.Name, CompletionItemKind.Module)); + break; + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(document.FilePath): + completions.Add(CompletionItemSource.CreateCompletionItem(moduleImport.Name, CompletionItemKind.Module)); + break; } } } diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index f39b825c9..17508fde5 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1013,7 +1013,7 @@ public async Task FromDotInRootWithInitPy() { var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); - result.Should().OnlyHaveLabels("module1", "__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); + result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); } [TestMethod, Priority(0)] @@ -1088,7 +1088,7 @@ public async Task FromDotInImplicitPackage() { [TestMethod, Priority(0)] public async Task SubmoduleMember() { var appUri = TestData.GetTestSpecificUri("app.py"); - await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "import m1\nx = 1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "from . import m1\nfrom . import m2\nx = 1"); await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m2", "__init__.py"), string.Empty); From f2d7f5345d3081c70e24b99017188bf78d6faaa8 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 18 Sep 2019 16:15:45 -0700 Subject: [PATCH 06/19] Formatting --- src/LanguageServer/Impl/Completion/ImportCompletion.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index e02c5264d..4449882ed 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -167,8 +167,8 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im if (module != null) { completions.AddRange(module.GetMemberNames() - .Where(n => !string.IsNullOrEmpty(n)) - .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); + .Where(n => !string.IsNullOrEmpty(n)) + .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); } if (importSearchResult is IImportChildrenSource children) { From b353eb9ef3f341119392924ab17c18e9192d1feb Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 10:17:12 -0700 Subject: [PATCH 07/19] Remove unused --- .../Impl/Extensions/PythonModuleExtensions.cs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index 7571031b0..e4036465d 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -74,39 +74,5 @@ internal static string GetComment(this IPythonModule module, int lineNum) { internal static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); internal static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); - - public static IEnumerable GetSubmoduleNames(this IPythonModule module) { - var searchResult = module.Interpreter.ModuleResolution.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(module.Name, 1), 0, true); - if (searchResult is IImportChildrenSource children) { - foreach (var name in children.GetChildrenNames()) { - if (children.TryGetChildImport(name, out var imports)) { - switch (imports) { - case ImplicitPackageImport packageImport: - yield return packageImport.Name; - break; - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): - yield return moduleImport.Name; - break; - } - } - } - } - } - - public static IPythonModule GetSubmodule(this IPythonModule module, string subModuleName) { - var mres = module.Interpreter.ModuleResolution; - var searchResult = mres.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(module.Name, 1), 0, true); - if (searchResult is IImportChildrenSource children) { - if (children.TryGetChildImport(subModuleName, out var imports)) { - switch (imports) { - case ImplicitPackageImport packageImport: - return mres.GetImportedModule(packageImport.FullName); - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): - return mres.GetImportedModule(moduleImport.FullName); - } - } - } - return null; - } } } From da787143fdaf127233b7ab3b7c9b11e74cee6146 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 12:50:09 -0700 Subject: [PATCH 08/19] Move filtering to completions Declare submodule as member in case of import A.B inside A --- .../Impl/Analyzer/Handlers/ImportHandler.cs | 33 +++-- .../Impl/Extensions/PythonModuleExtensions.cs | 17 +-- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 36 +----- src/Analysis/Ast/Test/ImportTests.cs | 19 ++- .../Impl/Completion/CompletionContext.cs | 5 +- .../Impl/Completion/CompletionSource.cs | 7 +- .../Impl/Completion/ExpressionCompletion.cs | 57 ++++++-- .../Impl/Completion/ModuleMembersFilter.cs | 63 +++++++++ .../Impl/Implementation/Server.cs | 2 +- src/LanguageServer/Test/CompletionTests.cs | 122 +++++++++--------- src/LanguageServer/Test/ImportsTests.cs | 40 +++--- 11 files changed, 251 insertions(+), 150 deletions(-) create mode 100644 src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index b18b14883..05f39353a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -55,9 +55,11 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar') // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; - var lastModule = default(PythonVariableModule); var firstModule = default(PythonVariableModule); - foreach (var nameExpression in moduleImportExpression.Names) { + var secondModule = default(PythonVariableModule); + var lastModule = default(PythonVariableModule); + for (var i = 0; i < moduleImportExpression.Names.Count; i++) { + var nameExpression = moduleImportExpression.Names[i]; importNames = importNames.Add(nameExpression.Name); var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute); if (!HandleImportSearchResult(imports, lastModule, asNameExpression, moduleImportExpression, out lastModule)) { @@ -65,16 +67,31 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa break; } + if (i == 1) { + secondModule = lastModule; + } + if (firstModule == default) { firstModule = lastModule; } } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') - // "import fob.oar.baz" is handled as fob = import_module('fob') if (!string.IsNullOrEmpty(asNameExpression?.Name) && lastModule != default) { Eval.DeclareVariable(asNameExpression.Name, lastModule, VariableSource.Import, asNameExpression); - } else if (firstModule != default && !string.IsNullOrEmpty(importNames[0])) { + return; + } + + // "import fob.oar.baz" when 'fob' is THIS module handled by declaring 'oar' as member. + // Consider pandas that has 'import pandas.testing' in __init__.py and 'testing' + // is available as member. See also https://github.com/microsoft/python-language-server/issues/1395 + if (firstModule?.Module == Eval.Module && importNames.Count > 1 && !string.IsNullOrEmpty(importNames[1]) && secondModule != default) { + Eval.DeclareVariable(importNames[1], secondModule, VariableSource.Import, moduleImportExpression.Names[1]); + return; + } + + // "import fob.oar.baz" is handled as fob = import_module('fob') + if (firstModule != default && !string.IsNullOrEmpty(importNames[0])) { var firstName = moduleImportExpression.Names[0]; Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, firstName); } @@ -92,7 +109,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV return TryGetPackageFromImport(packageImport, parent, out variableModule); case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); - Eval.ReportDiagnostics(Eval.Module.Uri, + Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; @@ -157,7 +174,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor return false; } } - + return true; } @@ -170,8 +187,8 @@ private void MakeUnresolvedImport(string variableName, string moduleName, Node l if (!string.IsNullOrEmpty(variableName)) { Eval.DeclareVariable(variableName, new SentinelModule(moduleName, Eval.Services), VariableSource.Import, location); } - Eval.ReportDiagnostics(Eval.Module.Uri, - new DiagnosticsEntry(Resources.ErrorUnresolvedImport.FormatInvariant(moduleName), + Eval.ReportDiagnostics(Eval.Module.Uri, + new DiagnosticsEntry(Resources.ErrorUnresolvedImport.FormatInvariant(moduleName), Eval.GetLocationInfo(location).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index e4036465d..3ffc6137b 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -13,12 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; @@ -43,12 +39,12 @@ internal static void AddAstNode(this IPythonModule module, object o, Node n) /// /// The line number internal static string GetLine(this IPythonModule module, int lineNum) { - string content = module.Analysis?.Document?.Content; + var content = module.Analysis?.Document?.Content; if (string.IsNullOrEmpty(content)) { return string.Empty; } - SourceLocation source = new SourceLocation(lineNum, 1); + var source = new SourceLocation(lineNum, 1); var start = module.GetAst().LocationToIndex(source); var end = start; @@ -62,9 +58,9 @@ internal static string GetLine(this IPythonModule module, int lineNum) { /// /// The line number internal static string GetComment(this IPythonModule module, int lineNum) { - string line = module.GetLine(lineNum); + var line = module.GetLine(lineNum); - int commentPos = line.IndexOf('#'); + var commentPos = line.IndexOf('#'); if (commentPos < 0) { return string.Empty; } @@ -72,7 +68,8 @@ internal static string GetComment(this IPythonModule module, int lineNum) { return line.Substring(commentPos + 1).Trim('\t', ' '); } - internal static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); - internal static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); + public static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); + public static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); + public static bool IsTypingModule(this IPythonModule module) => module?.ModuleType == ModuleType.Specialized && module.Name == "typing"; } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 8e52f8669..ec1e5dc1e 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,7 +22,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -159,40 +158,7 @@ public virtual string Documentation { #region IMemberContainer public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; - - public virtual IEnumerable GetMemberNames() { - var fs = Services.GetService(); - var thisModuleDirectory = Path.GetDirectoryName(FilePath); - - // drop imported modules and typing. - return GlobalScope.Variables - .Where(v => { - // Instances are always fine. - if (v.Value is IPythonInstance) { - return true; - } - - var valueType = v.Value?.GetPythonType(); - switch (valueType) { - case IPythonModule m: - // Do not re-export modules except submodules. - return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); - case IPythonFunctionType f when f.IsLambda(): - return false; - } - - if (this is TypingModule) { - return true; // Let typing module behave normally. - } - - // Do not re-export types from typing. However, do export variables - // assigned with types from typing. Example: - // from typing import Any # do NOT export Any - // x = Union[int, str] # DO export x - return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; - }) - .Select(v => v.Name); - } + public virtual IEnumerable GetMemberNames() => GlobalScope.Variables.Select(v => v.Name).ToArray(); #endregion #region ILocatedMember diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 0ce949174..92f6ed3d9 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -240,7 +240,7 @@ def exit(): } [TestMethod, Priority(0)] - public async Task ModuleMembers() { + public async Task ModuleInternalImportSys() { var appUri = TestData.GetTestSpecificUri("app.py"); await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "from . import m1\nimport sys"); await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); @@ -251,7 +251,22 @@ public async Task ModuleMembers() { await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1").And.NotHaveMember("sys"); + analysis.Should().HaveVariable("package").Which.Should().HaveMembers("m1", "sys"); + } + + [TestMethod, Priority(0)] + public async Task ModuleImportingSubmodule() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "__init__.py"), "import package.m1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "m1", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var doc = rdt.OpenDocument(appUri, "import package"); + + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1"); } } } diff --git a/src/LanguageServer/Impl/Completion/CompletionContext.cs b/src/LanguageServer/Impl/Completion/CompletionContext.cs index b40f8a44b..f71f5b944 100644 --- a/src/LanguageServer/Impl/Completion/CompletionContext.cs +++ b/src/LanguageServer/Impl/Completion/CompletionContext.cs @@ -15,6 +15,7 @@ using System.Linq; using Microsoft.Python.Analysis; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; @@ -28,12 +29,14 @@ internal sealed class CompletionContext { public int Position { get; } public TokenSource TokenSource => _ts ?? (_ts = new TokenSource(Analysis.Document, Position)); public CompletionItemSource ItemSource { get; } + public IServiceContainer Services { get; } - public CompletionContext(IDocumentAnalysis analysis, SourceLocation location, CompletionItemSource itemSource) { + public CompletionContext(IDocumentAnalysis analysis, SourceLocation location, CompletionItemSource itemSource, IServiceContainer services) { Location = location; Analysis = analysis; Position = Ast.LocationToIndex(location); ItemSource = itemSource; + Services = services; } public SourceLocation IndexToLocation(int index) => Ast.IndexToLocation(index); diff --git a/src/LanguageServer/Impl/Completion/CompletionSource.cs b/src/LanguageServer/Impl/Completion/CompletionSource.cs index 7f0a3f714..d8666c5a5 100644 --- a/src/LanguageServer/Impl/Completion/CompletionSource.cs +++ b/src/LanguageServer/Impl/Completion/CompletionSource.cs @@ -17,6 +17,7 @@ using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Analyzer.Expressions; using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -24,9 +25,11 @@ namespace Microsoft.Python.LanguageServer.Completion { internal sealed class CompletionSource { private readonly CompletionItemSource _itemSource; + private readonly IServiceContainer _services; - public CompletionSource(IDocumentationSource docSource, ServerSettings.PythonCompletionOptions completionSettings) { + public CompletionSource(IDocumentationSource docSource, ServerSettings.PythonCompletionOptions completionSettings, IServiceContainer services) { _itemSource = new CompletionItemSource(docSource, completionSettings); + _services = services; } public ServerSettings.PythonCompletionOptions Options { @@ -39,7 +42,7 @@ public CompletionResult GetCompletions(IDocumentAnalysis analysis, SourceLocatio return CompletionResult.Empty; } - var context = new CompletionContext(analysis, location, _itemSource); + var context = new CompletionContext(analysis, location, _itemSource, _services); ExpressionLocator.FindExpression(analysis.Ast, location, FindExpressionOptions.Complete, out var expression, out var statement, out var scope); diff --git a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs index 5e9ff8d33..c3cbc216c 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs @@ -19,7 +19,10 @@ using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; using System.Collections.Generic; +using System.IO; using System.Linq; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.IO; namespace Microsoft.Python.LanguageServer.Completion { internal static class ExpressionCompletion { @@ -41,23 +44,57 @@ private static IEnumerable GetItemsFromExpression(Expression e, if (!value.IsUnknown()) { var type = value.GetPythonType(); - if(type is IPythonClassType cls) { + if (type is IPythonClassType cls) { return GetClassItems(cls, e, context); } - var items = new List(); - foreach (var t in type.GetMemberNames().ToArray()) { - var m = type.GetMember(t); - if (m is IVariable v && v.Source != VariableSource.Declaration) { - continue; - } - items.Add(context.ItemSource.CreateCompletionItem(t, m, type)); - } - return items; + var memberNames = type is IPythonModule module + ? GetModuleMemberNames(module, context) + : type.GetMemberNames(); + + return memberNames.Select(name => context.ItemSource.CreateCompletionItem(name, type.GetMember(name), type)); } return Enumerable.Empty(); } + private static IEnumerable GetModuleMemberNames(IPythonModule module, CompletionContext context) { + var variables = module.Analysis?.GlobalScope?.Variables; + if (variables == null) { + return module.GetMemberNames(); + } + + var fs = context.Services.GetService(); + var thisModuleDirectory = Path.GetDirectoryName(module.FilePath); + // drop imported modules and typing. + return variables + .Where(v => { + // Instances are always fine. + if (v.Value is IPythonInstance) { + return true; + } + + var valueType = v.Value?.GetPythonType(); + switch (valueType) { + case IPythonModule m: + // Do not show modules except submodules. + return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); + case IPythonFunctionType f when f.IsLambda(): + return false; + } + + if (module.IsTypingModule()) { + return true; // Let typing module behave normally. + } + + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + return valueType?.DeclaringModule.IsTypingModule() != true || v.Name != valueType.Name; + }) + .Select(v => v.Name); + } + private static IEnumerable GetClassItems(IPythonClassType cls, Expression e, CompletionContext context) { var eval = context.Analysis.ExpressionEvaluator; // See if we are completing on self. Note that we may be inside inner function diff --git a/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs b/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs new file mode 100644 index 000000000..662adebcf --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs @@ -0,0 +1,63 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.IO; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ModuleMembersFilter { + public static IEnumerable GetMemberNames(this IPythonModule module, IFileSystem fs) { + if(!(module is IDocument document) || document.Analysis?.GlobalScope?.Variables == null) { + return module.GetMemberNames(); + } + + var thisModuleDirectory = Path.GetDirectoryName(module.FilePath); + // drop imported modules and typing. + return document.Analysis.GlobalScope.Variables + .Where(v => { + // Instances are always fine. + if (v.Value is IPythonInstance) { + return true; + } + + var valueType = v.Value?.GetPythonType(); + switch (valueType) { + case IPythonModule m: + // Do not show modules except submodules. + return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); + case IPythonFunctionType f when f.IsLambda(): + return false; + } + + if (module.IsTypingModule()) { + return true; // Let typing module behave normally. + } + + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + return valueType?.DeclaringModule.IsTypingModule() != true || v.Name != valueType.Name; + }) + .Select(v => v.Name); + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 01c643ea9..4a553a2f8 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -168,7 +168,7 @@ public async Task InitializedAsync(InitializedParams @params, CancellationToken _completionSource = new CompletionSource( ChooseDocumentationSource(textDocCaps?.completion?.completionItem?.documentationFormat), - Settings.completion + Settings.completion, Services ); _hoverSource = new HoverSource( diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index 17508fde5..b797ac499 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -59,7 +59,7 @@ def method(self): "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(8, 1)); comps.Should().HaveLabels("C", "x", "y", "while", "for"); } @@ -71,7 +71,7 @@ public async Task StringMembers() { x. "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(3, 3)); comps.Should().HaveLabels(@"isupper", @"capitalize", @"split"); } @@ -83,7 +83,7 @@ import datetime datetime.datetime. "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(3, 19)); comps.Should().HaveLabels("now", @"tzinfo", @"ctime"); } @@ -98,7 +98,7 @@ def method1(self): pass ABCDE.me "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(5, 4)); comps.Should().HaveLabels(@"ABCDE"); @@ -116,7 +116,7 @@ class oar(list): pass "; var analysis = await GetAnalysisAsync(code, version); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 9)); result.Should().HaveItem("append") @@ -131,7 +131,7 @@ class Test(): def __ "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 10)); result.Should().HaveItem("__init__") @@ -149,7 +149,7 @@ class oar(list): pass "; var analysis = await GetAnalysisAsync(code, version); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 9)); result.Should().HaveItem("append") @@ -168,7 +168,7 @@ class Test(A): def __ "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(7, 10)); result.Should().HaveItem("__init__") @@ -192,7 +192,7 @@ def fob(self): "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(4, 8)); result.Should().HaveItem("a"); } @@ -209,7 +209,7 @@ def oar(self, a): "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(4, 8)); result.Should().HaveItem("a"); } @@ -231,7 +231,7 @@ def oar(self): pass var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(11, 3)); result.Should().NotContainLabels("fob"); result.Should().HaveLabels("oar"); @@ -250,7 +250,7 @@ class B(A): def f"; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(7, 10)); result.Should() @@ -265,13 +265,13 @@ public async Task InRaise(bool is3X) { var version = is3X ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; var analysis = await GetAnalysisAsync("raise ", version); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); if (is3X) { analysis = await GetAnalysisAsync("raise Exception from ", PythonVersions.LatestAvailable3X); - cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); @@ -301,7 +301,7 @@ public async Task InRaise(bool is3X) { [TestMethod, Priority(0)] public async Task InExcept() { var analysis = await GetAnalysisAsync("try:\n pass\nexcept "); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 8)); result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); @@ -342,7 +342,7 @@ public async Task AfterDot() { x "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 3)); result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); @@ -369,7 +369,7 @@ public async Task AfterDot() { [TestMethod, Priority(0)] public async Task AfterAssign() { var analysis = await GetAnalysisAsync("x = x\ny = "); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(2, 4)); result.Should().HaveLabels("x", "abs"); @@ -389,7 +389,7 @@ def test_exception(self): self.assertRaises(TypeError). "; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(5, 38)); result.Should().HaveInsertTexts("exception"); @@ -401,7 +401,7 @@ public async Task WithWhitespaceAroundDot() { sys . version "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(2, 7)); result.Should().HaveLabels("argv"); @@ -410,7 +410,7 @@ sys . version [TestMethod, Priority(0)] public async Task MarkupKindValid() { var analysis = await GetAnalysisAsync("import sys\nsys.\n"); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(2, 5)); result.Completions?.Select(i => i.documentation?.kind).ExcludeDefault() @@ -427,7 +427,7 @@ from typing import NewType foo. "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(6, 5)); result.Should().HaveLabels("clear", "copy", "items", "keys", "update", "values"); @@ -444,7 +444,7 @@ def func(a: List[str]): pass "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(5, 7)); result.Should().HaveLabels("clear", "copy", "count", "index", "remove", "reverse"); @@ -464,7 +464,7 @@ def func(a: Dict[int, str]): pass "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(5, 7)); result.Should().HaveLabels("keys", "values"); @@ -494,7 +494,7 @@ def get(self) -> _T: y = boxedstr. "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(14, 14)); result.Should().HaveItem("get").Which.Should().HaveDocumentation("Box.get() -> int"); @@ -526,7 +526,7 @@ def get(self) -> _T: y = boxedstr. "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(14, 14)); result.Should().HaveLabels("append", "index"); @@ -555,7 +555,7 @@ def fob(self, x): def baz(self): pass "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var completionInD = cs.GetCompletions(analysis, new SourceLocation(3, 5)); var completionInOar = cs.GetCompletions(analysis, new SourceLocation(5, 9)); @@ -581,7 +581,7 @@ def abc(self): x.abc() "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var objectMemberNames = analysis.Document.Interpreter.GetBuiltinType(BuiltinTypeId.Object).GetMemberNames(); var completion = cs.GetCompletions(analysis, new SourceLocation(7, 1)); @@ -598,7 +598,7 @@ public async Task InFunctionDefinition(bool is3X) { var version = is3X ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; var analysis = await GetAnalysisAsync("def f(a, b:int, c=2, d:float=None): pass", version); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 5)); result.Should().HaveNoCompletion(); @@ -632,7 +632,7 @@ public async Task InFunctionDefinition(bool is3X) { [TestMethod, Priority(0)] public async Task InFunctionDefinition_2X() { var analysis = await GetAnalysisAsync("@dec" + Environment.NewLine + "def f(): pass", PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 1)); result.Should().HaveLabels("any"); @@ -656,7 +656,7 @@ public async Task InFunctionDefinition_2X() { [TestMethod, Priority(0)] public async Task InFunctionDefinition_3X() { var analysis = await GetAnalysisAsync("@dec" + Environment.NewLine + "async def f(): pass", PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 1)); result.Should().HaveLabels("any"); @@ -684,7 +684,7 @@ public async Task InClassDefinition(bool is3x) { var version = is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; var analysis = await GetAnalysisAsync("class C(object, parameter=MC): pass", version); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 8)); result.Should().HaveNoCompletion(); @@ -726,7 +726,7 @@ public async Task InClassDefinition(bool is3x) { [TestMethod, Priority(0)] public async Task InWithStatement() { var analysis = await GetAnalysisAsync("with x as y, z as w: pass"); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 6)); result.Should().HaveAnyCompletions(); @@ -780,7 +780,7 @@ public async Task ImportInPackage() { var analysis2 = await module2.GetAnalysisAsync(-1); var analysis3 = await module3.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis1, new SourceLocation(1, 16)); result.Should().OnlyHaveLabels("module2", "sub_package"); @@ -801,7 +801,7 @@ public async Task InImport() { "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(2, 7)); result.Should().HaveLabels("from", "import", "abs", "dir").And.NotContainLabels("abc"); @@ -887,7 +887,7 @@ def i(): pass def pass"; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 9)); result.Should().HaveNoCompletion(); @@ -905,7 +905,7 @@ def i(): pass public async Task NoCompletionInEllipsis(bool is2x) { const string code = "..."; var analysis = await GetAnalysisAsync(code, is2x ? PythonVersions.LatestAvailable2X : PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 4)); result.Should().HaveNoCompletion(); @@ -917,7 +917,7 @@ public async Task NoCompletionInEllipsis(bool is2x) { [DataTestMethod, Priority(0)] public async Task NoCompletionInString(bool is2x) { var analysis = await GetAnalysisAsync("\"str.\"", is2x ? PythonVersions.LatestAvailable2X : PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 6)); result.Should().HaveNoCompletion(); } @@ -925,7 +925,7 @@ public async Task NoCompletionInString(bool is2x) { [TestMethod, Priority(0)] public async Task NoCompletionInOpenString() { var analysis = await GetAnalysisAsync("'''."); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 5)); result.Should().HaveNoCompletion(); } @@ -936,7 +936,7 @@ public async Task NoCompletionInOpenString() { [DataTestMethod, Priority(0)] public async Task NoCompletionInFStringConstant(string openFString) { var analysis = await GetAnalysisAsync(openFString); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 5)); result.Should().HaveNoCompletion(); } @@ -944,7 +944,7 @@ public async Task NoCompletionInFStringConstant(string openFString) { [TestMethod, Priority(0)] public async Task NoCompletionBadImportExpression() { var analysis = await GetAnalysisAsync("import os,."); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); cs.GetCompletions(analysis, new SourceLocation(1, 12)); // Should not crash. } @@ -952,7 +952,7 @@ public async Task NoCompletionBadImportExpression() { public async Task NoCompletionInComment() { var analysis = await GetAnalysisAsync("x = 1 #str. more text"); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 12)); result.Should().HaveNoCompletion(); } @@ -966,7 +966,7 @@ import os os. "; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 4)); result.Should().HaveLabels("path", @"devnull", "SEEK_SET", @"curdir"); @@ -981,7 +981,7 @@ import os os.path. "; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 9)); result.Should().HaveLabels("split", @"getsize", @"islink", @"abspath"); @@ -991,7 +991,7 @@ import os public async Task FromDotInRoot() { const string code = "from ."; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().HaveNoCompletion(); @@ -1011,7 +1011,7 @@ public async Task FromDotInRootWithInitPy() { var analysis = await module1.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); } @@ -1034,7 +1034,7 @@ public async Task FromDotInExplicitPackage() { await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await module.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().HaveLabels("module2", "sub_package", "answer"); @@ -1057,7 +1057,7 @@ public async Task FromPartialName() { await module.GetAnalysisAsync(-1); var analysis1 = await module1.GetAnalysisAsync(-1); var analysis2 = await module2.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis1, new SourceLocation(1, 8)); result.Should().HaveLabels("package").And.NotContainLabels("module2", "sub_package", "answer"); @@ -1079,7 +1079,7 @@ public async Task FromDotInImplicitPackage() { rdt.OpenDocument(module3, string.Empty); var analysis = await module.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); result.Should().OnlyHaveLabels("module2", "sub_package"); @@ -1098,7 +1098,7 @@ public async Task SubmoduleMember() { await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(2, 9)); result.Should().HaveLabels("m1", "m2", "x"); @@ -1113,7 +1113,7 @@ from os.path import exists as EX E "; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 2)); result.Should().HaveLabels("EX"); @@ -1126,7 +1126,7 @@ from os.path import exists as EX public async Task NoDuplicateMembers() { const string code = @"import sy"; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 10)); result.Completions.Count(c => c.label.EqualsOrdinal(@"sys")).Should().Be(1); @@ -1143,7 +1143,7 @@ class A: ... a. "; var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var extraMembers = new[] { "mro", "__dict__", @"__weakref__" }; var result = cs.GetCompletions(analysis, new SourceLocation(4, 3)); if (is3x) { @@ -1170,7 +1170,7 @@ def main(req: func.HttpRequest) -> func.HttpResponse: $"'azure.functions' package is not installed for Python {ver}, see https://github.com/Microsoft/python-language-server/issues/462"); } - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(5, 23)); result.Should().HaveLabels("get"); result.Completions.First(x => x.label == "get").Should().HaveDocumentation("dict.get*"); @@ -1182,7 +1182,7 @@ public async Task InForEnumeration() { for a, b in x: "); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(3, 4)); result.Should().HaveLabels("a", "b"); } @@ -1195,7 +1195,7 @@ public async Task NoCompletionForCurrentModuleName(bool empty) { var code = empty ? string.Empty : $"{Path.GetFileNameWithoutExtension(modulePath)}."; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X, null, modulePath); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, code.Length + 1)); result.Should().NotContainLabels(analysis.Document.Name); } @@ -1213,7 +1213,7 @@ import sys var analysis = await GetDocumentAnalysisAsync(doc); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var completions = cs.GetCompletions(analysis, new SourceLocation(3, 5)); completions.Should().HaveLabels("argv", "path", "exit"); } @@ -1225,7 +1225,7 @@ def func(): aaa = 1 a"; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(4, 2)); result.Completions.Select(c => c.label).Should().NotContain("aaa"); @@ -1244,7 +1244,7 @@ def func(self): A(). "; var analysis = await GetAnalysisAsync(code); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(7, 14)); result.Completions.Select(c => c.label).Should().Contain("__x").And.NotContain("_A__x"); @@ -1270,7 +1270,7 @@ def test(x: Foo = func()): x. "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(13, 7)); comps.Should().HaveLabels("name", "z"); } @@ -1281,7 +1281,7 @@ public async Task AddBrackets() { var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); ServerSettings.completion.addBrackets = true; - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(1, 5)); var print = comps.Completions.FirstOrDefault(x => x.label == "print"); @@ -1313,7 +1313,7 @@ def method2(self): "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(11, 21)); var names = comps.Completions.Select(c => c.label); diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index fd3b4824b..b114e05ff 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -71,7 +71,7 @@ import projectB.foo.baz var doc = rdt.OpenDocument(new Uri(appPath), appCode, appPath); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(7, 10)); comps.Should().HaveLabels("foo"); @@ -117,7 +117,7 @@ from projectB.foo import baz var doc = rdt.OpenDocument(new Uri(appPath), appCode, appPath); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(8, 10)); comps.Should().HaveLabels("foo"); @@ -154,7 +154,7 @@ public async Task SysModuleChain() { await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await doc1.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 5)); comps.Should().HaveLabels("VALUE"); } @@ -176,7 +176,7 @@ await TestData.CreateTestSpecificFileAsync("module2.py", @"import sys await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 5)); comps.Should().HaveLabels("VALUE"); } @@ -202,7 +202,7 @@ public async Task UncSearchPaths() { var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(1, 21)); comps.Should().HaveLabels("module1", "module2"); @@ -257,7 +257,7 @@ def method2(): await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 6)); comps.Should().HaveLabels("A").And.NotContainLabels("B"); @@ -296,7 +296,7 @@ import package.sub_package.module2 var doc = rdt.OpenDocument(new Uri(appPath), appCode); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(5, 9)); comps.Should().OnlyHaveLabels("sub_package"); @@ -333,7 +333,7 @@ import package.module.submodule as submodule var doc = rdt.OpenDocument(appUri, appCode); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); comps.Should().HaveLabels("Y").And.NotContainLabels("X"); @@ -364,7 +364,7 @@ from package.module import submodule var doc = rdt.OpenDocument(appUri, appCode); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); comps.Should().HaveLabels("Y").And.NotContainLabels("X"); @@ -395,7 +395,7 @@ from .sub_package.module import submodule var doc = rdt.OpenDocument(new Uri(appPath), appCode); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 8)); comps.Should().HaveLabels("Y").And.NotContainLabels("X"); @@ -464,7 +464,7 @@ from module3 import A3 await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(10, 4)); comps.Should().HaveLabels("M2"); @@ -478,7 +478,7 @@ from module3 import A3 [TestMethod, Priority(0)] public async Task TypingModule() { var analysis = await GetAnalysisAsync(@"from typing import "); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(1, 20)); comps.Should().HaveLabels("TypeVar", "List", "Dict", "Union"); } @@ -503,7 +503,7 @@ public async Task RelativeImportsFromParent() { await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await module2.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 9)); comps.Should().HaveLabels("X"); } @@ -528,7 +528,7 @@ public async Task FromImport_ModuleAffectsPackage(string appCodeImport) { var doc = rdt.OpenDocument(new Uri(appPath), appCode1); var analysis = await doc.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 13)); comps.Should().OnlyHaveLabels("module"); @@ -581,7 +581,7 @@ from module1 import * await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 5)); comps.Should().HaveLabels("foo"); @@ -644,7 +644,7 @@ from module1 import * await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 5)); comps.Should().HaveLabels("foo"); @@ -707,7 +707,7 @@ from module1 import * await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(4, 5)); comps.Should().HaveLabels("foo"); @@ -751,7 +751,7 @@ from module1 import _B as B await analyzer.WaitForCompleteAnalysisAsync(); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(5, 5)); comps.Should().HaveLabels("foo"); @@ -790,7 +790,7 @@ public async Task Python2XRelativeImportInRoot() { var analysis = await app.GetAnalysisAsync(-1); var analysisInPackage = await appInPackage.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(2, 8)); comps.Should().HaveLabels("X"); @@ -824,7 +824,7 @@ import module2 var app = rdt.OpenDocument(appUri, appContent); var analysis = await app.GetAnalysisAsync(-1); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(3, 9)); comps.Should().HaveLabels("X"); From b67ab2ef58d47dd5bf75bd83b0109d6fae474e2c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 12:57:21 -0700 Subject: [PATCH 09/19] Add another test --- .../Impl/Analyzer/Handlers/ImportHandler.cs | 4 ++-- src/Analysis/Ast/Test/ImportTests.cs | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 05f39353a..5530e41a4 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -86,14 +86,14 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // Consider pandas that has 'import pandas.testing' in __init__.py and 'testing' // is available as member. See also https://github.com/microsoft/python-language-server/issues/1395 if (firstModule?.Module == Eval.Module && importNames.Count > 1 && !string.IsNullOrEmpty(importNames[1]) && secondModule != default) { + Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); Eval.DeclareVariable(importNames[1], secondModule, VariableSource.Import, moduleImportExpression.Names[1]); return; } // "import fob.oar.baz" is handled as fob = import_module('fob') if (firstModule != default && !string.IsNullOrEmpty(importNames[0])) { - var firstName = moduleImportExpression.Names[0]; - Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, firstName); + Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); } } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 92f6ed3d9..8a606ee84 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -268,5 +268,27 @@ public async Task ModuleImportingSubmodule() { var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); analysis.Should().HaveVariable("package").Which.Should().HaveMember("m1"); } + + [TestMethod, Priority(0)] + public async Task ModuleImportingSubmodules() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "__init__.py"), @" +from top import sub1 +import top.sub2 +import top.sub3.sub4 +"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub2.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub3", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub3", "sub4.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var doc = rdt.OpenDocument(appUri, "import top"); + + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + analysis.Should().HaveVariable("top").Which.Should().HaveMembers("sub1", "sub2", "sub3", "top"); + } } } From b2e1cf6db1509cdefc821d077738d20f28607272 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 13:17:50 -0700 Subject: [PATCH 10/19] using --- src/LanguageServer/Impl/LanguageServer.Lifetime.cs | 1 - src/LanguageServer/Impl/LanguageServer.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs index f09b44db4..3412109cc 100644 --- a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs +++ b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index ca4d1e962..9c838b868 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -15,7 +15,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -32,7 +31,6 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using StreamJsonRpc; -using Range = Microsoft.Python.Core.Text.Range; namespace Microsoft.Python.LanguageServer.Implementation { /// From 160148bea4389054ece48e409c098f96d7f43dc2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 13:32:58 -0700 Subject: [PATCH 11/19] De-duplicate --- .../Impl/Completion/ImportCompletion.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 4449882ed..0d02757f8 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -165,10 +165,9 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im completions.Add(CompletionItemSource.Star); } + var memberNames = (module?.GetMemberNames().Where(n => !string.IsNullOrEmpty(n)) ?? Enumerable.Empty()).ToHashSet(); if (module != null) { - completions.AddRange(module.GetMemberNames() - .Where(n => !string.IsNullOrEmpty(n)) - .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); + completions.AddRange(memberNames.Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n)))); } if (importSearchResult is IImportChildrenSource children) { @@ -177,14 +176,19 @@ private static CompletionResult GetResultFromImportSearch(IImportSearchResult im continue; } + string name = null; switch (imports) { case ImplicitPackageImport packageImport: - completions.Add(CompletionItemSource.CreateCompletionItem(packageImport.Name, CompletionItemKind.Module)); + name = packageImport.Name; break; case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(document.FilePath): - completions.Add(CompletionItemSource.CreateCompletionItem(moduleImport.Name, CompletionItemKind.Module)); + name = moduleImport.Name; break; } + + if (name != null && !memberNames.Contains(name)) { + completions.Add(CompletionItemSource.CreateCompletionItem(name, CompletionItemKind.Module)); + } } } From 53db4b74380de7742ad97287811df7d3695dcc2e Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 19 Sep 2019 13:50:14 -0700 Subject: [PATCH 12/19] Remove unused --- .../Impl/Completion/ModuleMembersFilter.cs | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs diff --git a/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs b/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs deleted file mode 100644 index 662adebcf..000000000 --- a/src/LanguageServer/Impl/Completion/ModuleMembersFilter.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Python.Analysis; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.IO; - -namespace Microsoft.Python.LanguageServer.Completion { - internal static class ModuleMembersFilter { - public static IEnumerable GetMemberNames(this IPythonModule module, IFileSystem fs) { - if(!(module is IDocument document) || document.Analysis?.GlobalScope?.Variables == null) { - return module.GetMemberNames(); - } - - var thisModuleDirectory = Path.GetDirectoryName(module.FilePath); - // drop imported modules and typing. - return document.Analysis.GlobalScope.Variables - .Where(v => { - // Instances are always fine. - if (v.Value is IPythonInstance) { - return true; - } - - var valueType = v.Value?.GetPythonType(); - switch (valueType) { - case IPythonModule m: - // Do not show modules except submodules. - return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); - case IPythonFunctionType f when f.IsLambda(): - return false; - } - - if (module.IsTypingModule()) { - return true; // Let typing module behave normally. - } - - // Do not re-export types from typing. However, do export variables - // assigned with types from typing. Example: - // from typing import Any # do NOT export Any - // x = Union[int, str] # DO export x - return valueType?.DeclaringModule.IsTypingModule() != true || v.Name != valueType.Name; - }) - .Select(v => v.Name); - } - } -} From 7659217729ead389e8e0c5d628320914206f7dee Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 23 Sep 2019 12:58:13 -0700 Subject: [PATCH 13/19] Revert to initial GetMemberNames/GetMember on module --- .../Impl/Extensions/PythonModuleExtensions.cs | 38 +++++++++++++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 36 ++++++++++++- .../Test/FluentAssertions/MemberAssertions.cs | 18 ++++++- .../FluentAssertions/VariableAssertions.cs | 5 ++ src/Analysis/Ast/Test/ImportTests.cs | 32 +++++++++++ .../Impl/Completion/ExpressionCompletion.cs | 54 +++---------------- src/LanguageServer/Test/CompletionTests.cs | 4 +- 7 files changed, 133 insertions(+), 54 deletions(-) diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index 3ffc6137b..2a3f0d34f 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -13,8 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; @@ -71,5 +75,39 @@ internal static string GetComment(this IPythonModule module, int lineNum) { public static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); public static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); public static bool IsTypingModule(this IPythonModule module) => module?.ModuleType == ModuleType.Specialized && module.Name == "typing"; + + public static IEnumerable GetSubmoduleNames(this IPythonModule module) { + var searchResult = module.Interpreter.ModuleResolution.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Empty(), 1, true); + if (searchResult is IImportChildrenSource children) { + foreach (var name in children.GetChildrenNames()) { + if (children.TryGetChildImport(name, out var imports)) { + switch (imports) { + case ImplicitPackageImport packageImport: + yield return packageImport.Name; + break; + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): + yield return moduleImport.Name; + break; + } + } + } + } + } + + public static IPythonModule GetSubmodule(this IPythonModule module, string subModuleName) { + var mres = module.Interpreter.ModuleResolution; + var searchResult = mres.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(subModuleName, 1), 1, true); + if (searchResult is IImportChildrenSource children) { + if (children.TryGetChildImport(subModuleName, out var imports)) { + switch (imports) { + case ImplicitPackageImport packageImport: + return mres.GetImportedModule(packageImport.FullName); + case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): + return mres.GetImportedModule(moduleImport.FullName); + } + } + } + return null; + } } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 5568db81c..2955a8b65 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -157,8 +157,40 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; - public virtual IEnumerable GetMemberNames() => GlobalScope.Variables.Select(v => v.Name).ToArray(); + public virtual IMember GetMember(string name) + => GlobalScope.Variables[name]?.Value ?? this.GetSubmodule(name); + public virtual IEnumerable GetMemberNames() { + var submoduleNames = this.GetSubmoduleNames(); + // drop imported modules and typing. + var memberNames = GlobalScope.Variables + .Where(v => { + // Instances are always fine. + if (v.Value is IPythonInstance) { + return true; + } + + var valueType = v.Value?.GetPythonType(); + switch (valueType) { + case PythonModule _: + case IPythonFunctionType f when f.IsLambda(): + return false; // Do not re-export modules. + } + + if (this is TypingModule) { + return true; // Let typing module behave normally. + } + + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; + }) + .Select(v => v.Name) + .ToArray(); + + return memberNames.Concat(submoduleNames.Except(memberNames)); + } #endregion #region ILocatedMember diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 871e0a37a..1f6b10c08 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -83,6 +83,20 @@ public AndWhichConstraint HaveBase(string name, s public AndWhichConstraint HaveMethod(string name, string because = "", params object[] reasonArgs) => HaveMember(name, because, reasonArgs).OfMemberType(PythonMemberType.Method); + public void HaveMemberName(string name, string because = "", params object[] reasonArgs) { + NotBeNull(); + + var t = Subject.GetPythonType(); + var mc = (IMemberContainer)t; + Execute.Assertion.ForCondition(mc != null) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {GetName(t)} to be a member container{{reason}}."); + + Execute.Assertion.ForCondition(mc.GetMemberNames().Contains(name)) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {GetName(t)} to have a member named '{name}'{{reason}}."); + } + public AndWhichConstraint HaveMember(string name, string because = "", params object[] reasonArgs) where TMember : class, IMember { @@ -125,7 +139,7 @@ public void HaveSameMembersAs(IMember other) { Debug.Assert(missingNames.Length == 0); missingNames.Should().BeEmpty("Subject has missing names: ", missingNames); - + Debug.Assert(extraNames.Length == 0); extraNames.Should().BeEmpty("Subject has extra names: ", extraNames); @@ -147,7 +161,7 @@ public void HaveSameMembersAs(IMember other) { var otherClass = otherMemberType as IPythonClassType; otherClass.Should().NotBeNull(); - if(subjectClass is IGenericType gt) { + if (subjectClass is IGenericType gt) { otherClass.Should().BeAssignableTo(); otherClass.IsGeneric.Should().Be(gt.IsGeneric, $"Class name: {subjectClass.Name}"); } diff --git a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs index 6758d2a09..fff01147f 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs @@ -89,6 +89,11 @@ public AndWhichConstraint HaveMember(string name, stri return new AndWhichConstraint(this, m); } + public void HaveMemberName(string name, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + Value.GetPythonType().GetMemberNames().Should().Contain(name); + } + public AndWhichConstraint HaveOverloadWithParametersAt(int index, string because = "", params object[] reasonArgs) { var constraint = HaveOverloadAt(index); var overload = constraint.Which; diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 4317e628c..6fdabd7c1 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -307,5 +307,37 @@ public async Task ImportPackageNoInitPy() { .Which.Should().HaveType().Which; sub1.Value.MemberType.Should().NotBe(ModuleType.Unresolved); } + + [TestMethod, Priority(0)] + public async Task DeepSubmoduleImport() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "sub2", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "sub2", "sub3", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "sub2", "sub3", "sub4", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var appDoc = rdt.OpenDocument(appUri, "import top.sub1.sub2.sub3.sub4"); + + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); + + var topModule = analysis.Should().HaveVariable("top") + .Which.Should().HaveType().Which; + + topModule.Should().HaveMemberName("sub1"); + var sub1Module = topModule.Should().HaveMember("sub1").Which; + + sub1Module.Should().HaveMemberName("sub2"); + var sub2Module = sub1Module.Should().HaveMember("sub2").Which; + + sub2Module.Should().HaveMemberName("sub3"); + var sub3Module = sub2Module.Should().HaveMember("sub3").Which; + + sub3Module.Should().HaveMemberName("sub4"); + sub3Module.Should().HaveMember("sub4"); + } } } diff --git a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs index c3cbc216c..9661cd263 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs @@ -13,16 +13,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using System.IO; +using System.Linq; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.IO; using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Core.IO; namespace Microsoft.Python.LanguageServer.Completion { internal static class ExpressionCompletion { @@ -48,53 +47,12 @@ private static IEnumerable GetItemsFromExpression(Expression e, return GetClassItems(cls, e, context); } - var memberNames = type is IPythonModule module - ? GetModuleMemberNames(module, context) - : type.GetMemberNames(); - - return memberNames.Select(name => context.ItemSource.CreateCompletionItem(name, type.GetMember(name), type)); + return type.GetMemberNames() + .Select(name => context.ItemSource.CreateCompletionItem(name, type.GetMember(name), type)); } return Enumerable.Empty(); } - private static IEnumerable GetModuleMemberNames(IPythonModule module, CompletionContext context) { - var variables = module.Analysis?.GlobalScope?.Variables; - if (variables == null) { - return module.GetMemberNames(); - } - - var fs = context.Services.GetService(); - var thisModuleDirectory = Path.GetDirectoryName(module.FilePath); - // drop imported modules and typing. - return variables - .Where(v => { - // Instances are always fine. - if (v.Value is IPythonInstance) { - return true; - } - - var valueType = v.Value?.GetPythonType(); - switch (valueType) { - case IPythonModule m: - // Do not show modules except submodules. - return !string.IsNullOrEmpty(m.FilePath) && fs.IsPathUnderRoot(thisModuleDirectory, Path.GetDirectoryName(m.FilePath)); - case IPythonFunctionType f when f.IsLambda(): - return false; - } - - if (module.IsTypingModule()) { - return true; // Let typing module behave normally. - } - - // Do not re-export types from typing. However, do export variables - // assigned with types from typing. Example: - // from typing import Any # do NOT export Any - // x = Union[int, str] # DO export x - return valueType?.DeclaringModule.IsTypingModule() != true || v.Name != valueType.Name; - }) - .Select(v => v.Name); - } - private static IEnumerable GetClassItems(IPythonClassType cls, Expression e, CompletionContext context) { var eval = context.Analysis.ExpressionEvaluator; // See if we are completing on self. Note that we may be inside inner function diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index be560105b..e97ba71df 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1013,7 +1013,7 @@ public async Task FromDotInRootWithInitPy() { var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); - result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); + result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__", "module1"); } [TestMethod, Priority(0)] @@ -1336,7 +1336,7 @@ public async Task FromImportPackageNoInitPy() { await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var comps = cs.GetCompletions(analysis, new SourceLocation(1, 18)); var names = comps.Completions.Select(c => c.label); names.Should().Contain(new[] { "sub1" }); From 75ac081ebe7a6b58c68c1b72a41fd1e5df3812bc Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 23 Sep 2019 15:49:35 -0700 Subject: [PATCH 14/19] Prefer submodule over variable --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 2955a8b65..2820f22a3 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -158,7 +158,8 @@ public virtual string Documentation { #region IMemberContainer public virtual IMember GetMember(string name) - => GlobalScope.Variables[name]?.Value ?? this.GetSubmodule(name); + => this.GetSubmodule(name) ?? GlobalScope.Variables[name]?.Value; + public virtual IEnumerable GetMemberNames() { var submoduleNames = this.GetSubmoduleNames(); // drop imported modules and typing. From 4f90bcfaa63517ff0fdf7730aaa5f78e5c2b5fbc Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 23 Sep 2019 21:50:11 -0700 Subject: [PATCH 15/19] Use child modules instead --- .../Impl/Analyzer/Handlers/ImportHandler.cs | 30 ++++++++-------- .../Impl/Extensions/PythonModuleExtensions.cs | 34 ------------------- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 8 ++--- .../Ast/Impl/Modules/PythonVariableModule.cs | 4 +-- src/LanguageServer/Test/CompletionTests.cs | 2 +- 5 files changed, 21 insertions(+), 57 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 706bf0d4d..2987c1788 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -55,9 +55,8 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar') // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; - var firstModule = default(PythonVariableModule); - var secondModule = default(PythonVariableModule); var lastModule = default(PythonVariableModule); + var modules = new PythonVariableModule[moduleImportExpression.Names.Count]; for (var i = 0; i < moduleImportExpression.Names.Count; i++) { var nameExpression = moduleImportExpression.Names[i]; importNames = importNames.Add(nameExpression.Name); @@ -66,14 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa lastModule = default; break; } - - if (i == 1) { - secondModule = lastModule; - } - - if (firstModule == null) { - firstModule = lastModule; - } + modules[i] = lastModule; } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') @@ -82,18 +74,28 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa return; } + var firstModule = modules.Length > 0 ? modules[0] : null; + var secondModule = modules.Length > 1 ? modules[1] : null; + // "import fob.oar.baz" when 'fob' is THIS module handled by declaring 'oar' as member. // Consider pandas that has 'import pandas.testing' in __init__.py and 'testing' // is available as member. See also https://github.com/microsoft/python-language-server/issues/1395 if (firstModule?.Module == Eval.Module && importNames.Count > 1 && !string.IsNullOrEmpty(importNames[1]) && secondModule != null) { Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); Eval.DeclareVariable(importNames[1], secondModule, VariableSource.Import, moduleImportExpression.Names[1]); - return; + } else { + // "import fob.oar.baz" is handled as fob = import_module('fob') + if (firstModule != null && !string.IsNullOrEmpty(importNames[0])) { + Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); + } } - // "import fob.oar.baz" is handled as fob = import_module('fob') - if (firstModule != null && !string.IsNullOrEmpty(importNames[0])) { - Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); + // import a.b.c.d => declares a, b in the current module, c in b, d in c. + for (var i = 1; i < modules.Length - 1; i++) { + var child = modules[i + 1]; + if (child != null) { + modules[i]?.AddChildModule(child.Name, child); + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index 2a3f0d34f..c05e062c3 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -75,39 +75,5 @@ internal static string GetComment(this IPythonModule module, int lineNum) { public static bool IsNonUserFile(this IPythonModule module) => module.ModuleType.IsNonUserFile(); public static bool IsCompiled(this IPythonModule module) => module.ModuleType.IsCompiled(); public static bool IsTypingModule(this IPythonModule module) => module?.ModuleType == ModuleType.Specialized && module.Name == "typing"; - - public static IEnumerable GetSubmoduleNames(this IPythonModule module) { - var searchResult = module.Interpreter.ModuleResolution.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Empty(), 1, true); - if (searchResult is IImportChildrenSource children) { - foreach (var name in children.GetChildrenNames()) { - if (children.TryGetChildImport(name, out var imports)) { - switch (imports) { - case ImplicitPackageImport packageImport: - yield return packageImport.Name; - break; - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): - yield return moduleImport.Name; - break; - } - } - } - } - } - - public static IPythonModule GetSubmodule(this IPythonModule module, string subModuleName) { - var mres = module.Interpreter.ModuleResolution; - var searchResult = mres.CurrentPathResolver.FindImports(module.FilePath, Enumerable.Repeat(subModuleName, 1), 1, true); - if (searchResult is IImportChildrenSource children) { - if (children.TryGetChildImport(subModuleName, out var imports)) { - switch (imports) { - case ImplicitPackageImport packageImport: - return mres.GetImportedModule(packageImport.FullName); - case ModuleImport moduleImport when !moduleImport.ModulePath.PathEquals(module.FilePath): - return mres.GetImportedModule(moduleImport.FullName); - } - } - } - return null; - } } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 2820f22a3..92de189dd 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -157,13 +157,11 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) - => this.GetSubmodule(name) ?? GlobalScope.Variables[name]?.Value; + public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; public virtual IEnumerable GetMemberNames() { - var submoduleNames = this.GetSubmoduleNames(); // drop imported modules and typing. - var memberNames = GlobalScope.Variables + return GlobalScope.Variables .Where(v => { // Instances are always fine. if (v.Value is IPythonInstance) { @@ -189,8 +187,6 @@ public virtual IEnumerable GetMemberNames() { }) .Select(v => v.Name) .ToArray(); - - return memberNames.Concat(submoduleNames.Except(memberNames)); } #endregion diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index 70e4c9fab..6641a3c8a 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -67,8 +67,8 @@ public PythonVariableModule(IPythonModule module): base(module) { public void AddChildModule(string memberName, PythonVariableModule module) => _children[memberName] = module; - public IMember GetMember(string name) => Module?.GetMember(name) ?? (_children.TryGetValue(name, out var module) ? module : default); - public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(_children.Keys) : _children.Keys; + public IMember GetMember(string name) => _children.TryGetValue(name, out var module) ? module : Module?.GetMember(name); + public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(_children.Keys).Distinct() : _children.Keys; public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName); public IMember Index(IPythonInstance instance, IArgumentSet args) => Interpreter.UnknownType; diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index e97ba71df..978aa99de 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1013,7 +1013,7 @@ public async Task FromDotInRootWithInitPy() { var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); var result = cs.GetCompletions(analysis, new SourceLocation(1, 7)); - result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__", "module1"); + result.Should().OnlyHaveLabels("__dict__", "__file__", "__doc__", "__package__", "__debug__", "__name__", "__path__", "__spec__"); } [TestMethod, Priority(0)] From 4faafe112c125683ad580a999fdee9cd843fcf5a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 24 Sep 2019 12:06:00 -0700 Subject: [PATCH 16/19] Extra checks --- src/Analysis/Ast/Test/ScrapeTests.cs | 1 + .../DependencyResolution/PathResolverSnapshot.cs | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index b3b854470..d3278475d 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -274,6 +274,7 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params var pathResolver = interpreter.ModuleResolution.CurrentPathResolver; var modules = pathResolver.GetAllImportableModuleNames() .Select(n => pathResolver.GetModuleImportFromModuleName(n)) + .ExcludeDefault() .Where(i => i.RootPath.PathEquals(configuration.SitePackagesPath) || i.RootPath.PathEquals(configuration.LibraryPath)) .ToList(); diff --git a/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs b/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs index 1dd003ddb..68aa0f8c7 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs @@ -88,11 +88,14 @@ public ImmutableArray GetAllImportableModuleNames(bool includeImplicitPa while (items.Count > 0) { var item = items.Dequeue(); - if (!string.IsNullOrEmpty(item.FullModuleName) && (item.IsModule || includeImplicitPackages)) { - names = names.Add(item.FullModuleName); - } - foreach (var child in item.Children) { - items.Enqueue(child); + if (item != null) { + if (!string.IsNullOrEmpty(item.FullModuleName) && (item.IsModule || includeImplicitPackages)) { + names = names.Add(item.FullModuleName); + } + + foreach (var child in item.Children.ExcludeDefault()) { + items.Enqueue(child); + } } } From 1eb6410a9c9331c90c81a779df88a9f182be115a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 24 Sep 2019 13:24:48 -0700 Subject: [PATCH 17/19] Fix child module naming --- .../Ast/Impl/Analyzer/Handlers/ImportHandler.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 2987c1788..9aa16e12f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -56,7 +56,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; var lastModule = default(PythonVariableModule); - var modules = new PythonVariableModule[moduleImportExpression.Names.Count]; + var modules = new (string, PythonVariableModule)[moduleImportExpression.Names.Count]; for (var i = 0; i < moduleImportExpression.Names.Count; i++) { var nameExpression = moduleImportExpression.Names[i]; importNames = importNames.Add(nameExpression.Name); @@ -65,7 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa lastModule = default; break; } - modules[i] = lastModule; + modules[i] = (nameExpression.Name, lastModule); } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') @@ -74,8 +74,8 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa return; } - var firstModule = modules.Length > 0 ? modules[0] : null; - var secondModule = modules.Length > 1 ? modules[1] : null; + var firstModule = modules.Length > 0 ? modules[0].Item2 : null; + var secondModule = modules.Length > 1 ? modules[1].Item2 : null; // "import fob.oar.baz" when 'fob' is THIS module handled by declaring 'oar' as member. // Consider pandas that has 'import pandas.testing' in __init__.py and 'testing' @@ -92,9 +92,10 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import a.b.c.d => declares a, b in the current module, c in b, d in c. for (var i = 1; i < modules.Length - 1; i++) { - var child = modules[i + 1]; - if (child != null) { - modules[i]?.AddChildModule(child.Name, child); + var (childName, childModule) = modules[i + 1]; + if (!string.IsNullOrEmpty(childName) && childModule != null) { + var parent = modules[i].Item2; + parent?.AddChildModule(childName, childModule); } } } From 45825e1df6cbb2d2901e3b98427e57571dae571b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 24 Sep 2019 13:58:29 -0700 Subject: [PATCH 18/19] Prefer submodules to variables. --- .../Impl/Analyzer/Handlers/FromImportHandler.cs | 14 +++++++++----- .../Ast/Impl/Analyzer/Handlers/ImportHandler.cs | 14 +++++++------- src/Analysis/Ast/Test/ImportTests.cs | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 3374d04b2..ed9d4b5d8 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -69,9 +69,13 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor var nameExpression = asNames[i] ?? names[i]; var variableName = nameExpression?.Name ?? memberName; if (!string.IsNullOrEmpty(variableName)) { - var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - var exported = variable ?? variableModule.GetMember(memberName); - var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + // First try imports since child modules should win, i.e. in 'from a.b import c' + // 'c' should be a submodule if 'b' has one, even if 'b' also declares 'c = 1'. + var value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + // Now try exported + value = value ?? variableModule.GetMember(memberName); + // If nothing is exported, variables are still accessible. + value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; // Do not allow imported variables to override local declarations Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex)); } @@ -131,7 +135,7 @@ private bool CanOverwriteVariable(string name, int importPosition) { private IMember GetValueFromImports(PythonVariableModule parentModule, IImportChildrenSource childrenSource, string memberName) { if (childrenSource == null || !childrenSource.TryGetChildImport(memberName, out var childImport)) { - return Interpreter.UnknownType; + return null; } switch (childImport) { @@ -141,7 +145,7 @@ private IMember GetValueFromImports(PythonVariableModule parentModule, IImportCh case ImplicitPackageImport packageImport: return GetOrCreateVariableModule(packageImport.FullName, parentModule, memberName); default: - return Interpreter.UnknownType; + return null; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 9aa16e12f..62170c8be 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -56,7 +56,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; var lastModule = default(PythonVariableModule); - var modules = new (string, PythonVariableModule)[moduleImportExpression.Names.Count]; + var resolvedModules = new (string name, PythonVariableModule module)[moduleImportExpression.Names.Count]; for (var i = 0; i < moduleImportExpression.Names.Count; i++) { var nameExpression = moduleImportExpression.Names[i]; importNames = importNames.Add(nameExpression.Name); @@ -65,7 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa lastModule = default; break; } - modules[i] = (nameExpression.Name, lastModule); + resolvedModules[i] = (nameExpression.Name, lastModule); } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') @@ -74,8 +74,8 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa return; } - var firstModule = modules.Length > 0 ? modules[0].Item2 : null; - var secondModule = modules.Length > 1 ? modules[1].Item2 : null; + var firstModule = resolvedModules.Length > 0 ? resolvedModules[0].module : null; + var secondModule = resolvedModules.Length > 1 ? resolvedModules[1].module : null; // "import fob.oar.baz" when 'fob' is THIS module handled by declaring 'oar' as member. // Consider pandas that has 'import pandas.testing' in __init__.py and 'testing' @@ -91,10 +91,10 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa } // import a.b.c.d => declares a, b in the current module, c in b, d in c. - for (var i = 1; i < modules.Length - 1; i++) { - var (childName, childModule) = modules[i + 1]; + for (var i = 1; i < resolvedModules.Length - 1; i++) { + var (childName, childModule) = resolvedModules[i + 1]; if (!string.IsNullOrEmpty(childName) && childModule != null) { - var parent = modules[i].Item2; + var parent = resolvedModules[i].module; parent?.AddChildModule(childName, childModule); } } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 6fdabd7c1..955cd883f 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -339,5 +340,21 @@ public async Task DeepSubmoduleImport() { sub3Module.Should().HaveMemberName("sub4"); sub3Module.Should().HaveMember("sub4"); } + + [TestMethod, Priority(0)] + public async Task SubmoduleOverridesVariable() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "__init__.py"), "sub2 = 1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "sub2", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var appDoc = rdt.OpenDocument(appUri, "from top.sub1 import sub2"); + + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); + analysis.Should().HaveVariable("sub2").Which.Should().HaveType(BuiltinTypeId.Module); + } } } From 2cc0ee3de23a0f819e668afc9d94eb6d98c92430 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 24 Sep 2019 15:12:18 -0700 Subject: [PATCH 19/19] Prefer submodules to variables in import star --- .../Analyzer/Handlers/FromImportHandler.cs | 58 ++++++++++--------- src/Analysis/Ast/Test/ImportTests.cs | 16 +++++ 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index ed9d4b5d8..7975f97d2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -59,7 +59,7 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor // TODO: warn this is not a good style per // TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module // TODO: warn this is invalid if not in the global scope. - HandleModuleImportStar(variableModule, imports is ImplicitPackageImport, node.StartIndex); + HandleModuleImportStar(variableModule, imports, node.StartIndex, names[0]); return; } @@ -69,47 +69,51 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor var nameExpression = asNames[i] ?? names[i]; var variableName = nameExpression?.Name ?? memberName; if (!string.IsNullOrEmpty(variableName)) { - // First try imports since child modules should win, i.e. in 'from a.b import c' - // 'c' should be a submodule if 'b' has one, even if 'b' also declares 'c = 1'. - var value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); - // Now try exported - value = value ?? variableModule.GetMember(memberName); - // If nothing is exported, variables are still accessible. - value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; - // Do not allow imported variables to override local declarations - Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex)); + DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression); } } } } - private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage, int importPosition) { + private void HandleModuleImportStar(PythonVariableModule variableModule, IImportSearchResult imports, int importPosition, NameExpression nameExpression) { if (variableModule.Module == Module) { // from self import * won't define any new members return; } - // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. - var memberNames = isImplicitPackage + var memberNames = imports is ImplicitPackageImport ? variableModule.GetMemberNames() : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); foreach (var memberName in memberNames) { - var member = variableModule.GetMember(memberName); - if (member == null) { - Log?.Log(TraceEventType.Verbose, $"Undefined import: {variableModule.Name}, {memberName}"); - } else if (member.MemberType == PythonMemberType.Unknown) { - Log?.Log(TraceEventType.Verbose, $"Unknown import: {variableModule.Name}, {memberName}"); - } - - member = member ?? Eval.UnknownType; - if (member is IPythonModule m) { - ModuleResolution.GetOrLoadModule(m.Name); - } + DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression); + } + } - var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - // Do not allow imported variables to override local declarations - Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, Eval.DefaultLocation, CanOverwriteVariable(memberName, importPosition)); + /// + /// Determines value of the variable and declares it. Value depends if source module has submodule + /// that is named the same as the variable and/or it has internal variables named same as the submodule. + /// + /// 'from a.b import c' when 'c' is both submodule of 'b' and a variable declared inside 'b'. + /// Source module of the variable such as 'a.b' in 'from a.b import c as d'. + /// Module member name such as 'c' in 'from a.b import c as d'. + /// Import search result. + /// Name of the variable to declare, such as 'd' in 'from a.b import c as d'. + /// Position of the import statement. + /// Name expression of the variable. + private void DeclareVariable(PythonVariableModule variableModule, string memberName, IImportSearchResult imports, string variableName, int importPosition, Node nameExpression) { + // First try imports since child modules should win, i.e. in 'from a.b import c' + // 'c' should be a submodule if 'b' has one, even if 'b' also declares 'c = 1'. + var value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); + // Now try exported + value = value ?? variableModule.GetMember(memberName); + // If nothing is exported, variables are still accessible. + value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; + // Do not allow imported variables to override local declarations + Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, importPosition)); + // Make sure module is loaded and analyzed. + if (value is IPythonModule m) { + ModuleResolution.GetOrLoadModule(m.Name); } } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 955cd883f..2b7ce0c7c 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -356,5 +356,21 @@ public async Task SubmoduleOverridesVariable() { var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); analysis.Should().HaveVariable("sub2").Which.Should().HaveType(BuiltinTypeId.Module); } + + [TestMethod, Priority(0)] + public async Task SubmoduleOverridesVariableStarImport() { + var appUri = TestData.GetTestSpecificUri("app.py"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "__init__.py"), string.Empty); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "__init__.py"), "sub2 = 1"); + await TestData.CreateTestSpecificFileAsync(Path.Combine("top", "sub1", "sub2", "__init__.py"), string.Empty); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var appDoc = rdt.OpenDocument(appUri, "from top.sub1 import *"); + + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); + analysis.Should().HaveVariable("sub2").Which.Should().HaveType(BuiltinTypeId.Module); + } } }