diff --git a/package.json b/package.json index ab0f4b1bb82d..8fe6ab5c57c8 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "terminalDataWriteEvent", "terminalExecuteCommandEvent", "contribIssueReporter", + "notebookVariableProvider", "codeActionAI" ], "author": { diff --git a/python_files/get_variable_info.py b/python_files/get_variable_info.py new file mode 100644 index 000000000000..d60795982617 --- /dev/null +++ b/python_files/get_variable_info.py @@ -0,0 +1,539 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the project root +# for license information. + +import locale +import sys +from typing import ClassVar + + +# this class is from in ptvsd/debugpy tools +class SafeRepr(object): # noqa: UP004 + # Can be used to override the encoding from locale.getpreferredencoding() + locale_preferred_encoding = None + + # Can be used to override the encoding used for sys.stdout.encoding + sys_stdout_encoding = None + + # String types are truncated to maxstring_outer when at the outer- + # most level, and truncated to maxstring_inner characters inside + # collections. + maxstring_outer = 2**16 + maxstring_inner = 128 + string_types = (str, bytes) + bytes = bytes + set_info = (set, "{", "}", False) + frozenset_info = (frozenset, "frozenset({", "})", False) + int_types = (int,) + long_iter_types = (list, tuple, bytearray, range, dict, set, frozenset) + + # Collection types are recursively iterated for each limit in + # maxcollection. + maxcollection = (60, 20) + + # Specifies type, prefix string, suffix string, and whether to include a + # comma if there is only one element. (Using a sequence rather than a + # mapping because we use isinstance() to determine the matching type.) + collection_types = [ # noqa: RUF012 + (tuple, "(", ")", True), + (list, "[", "]", False), + frozenset_info, + set_info, + ] + try: + from collections import deque + + collection_types.append((deque, "deque([", "])", False)) + except Exception: + pass + + # type, prefix string, suffix string, item prefix string, + # item key/value separator, item suffix string + dict_types: ClassVar[list] = [(dict, "{", "}", "", ": ", "")] + try: + from collections import OrderedDict + + dict_types.append((OrderedDict, "OrderedDict([", "])", "(", ", ", ")")) + except Exception: + pass + + # All other types are treated identically to strings, but using + # different limits. + maxother_outer = 2**16 + maxother_inner = 128 + + convert_to_hex = False + raw_value = False + + def __call__(self, obj): + """ + :param object obj: + The object for which we want a representation. + + :return str: + Returns bytes encoded as utf-8 on py2 and str on py3. + """ # noqa: D205 + try: + return "".join(self._repr(obj, 0)) + except Exception: + try: + return f"An exception was raised: {sys.exc_info()[1]!r}" + except Exception: + return "An exception was raised" + + def _repr(self, obj, level): + """Returns an iterable of the parts in the final repr string.""" + try: + obj_repr = type(obj).__repr__ + except Exception: + obj_repr = None + + def has_obj_repr(t): + r = t.__repr__ + try: + return obj_repr == r + except Exception: + return obj_repr is r + + for t, prefix, suffix, comma in self.collection_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_iter(obj, level, prefix, suffix, comma) + + for ( + t, + prefix, + suffix, + item_prefix, + item_sep, + item_suffix, + ) in self.dict_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_dict( + obj, level, prefix, suffix, item_prefix, item_sep, item_suffix + ) + + for t in self.string_types: + if isinstance(obj, t) and has_obj_repr(t): + return self._repr_str(obj, level) + + if self._is_long_iter(obj): + return self._repr_long_iter(obj) + + return self._repr_other(obj, level) + + # Determines whether an iterable exceeds the limits set in + # maxlimits, and is therefore unsafe to repr(). + def _is_long_iter(self, obj, level=0): + try: + # Strings have their own limits (and do not nest). Because + # they don't have __iter__ in 2.x, this check goes before + # the next one. + if isinstance(obj, self.string_types): + return len(obj) > self.maxstring_inner + + # If it's not an iterable (and not a string), it's fine. + if not hasattr(obj, "__iter__"): + return False + + # If it's not an instance of these collection types then it + # is fine. Note: this is a fix for + # https://github.com/Microsoft/ptvsd/issues/406 + if not isinstance(obj, self.long_iter_types): + return False + + # Iterable is its own iterator - this is a one-off iterable + # like generator or enumerate(). We can't really count that, + # but repr() for these should not include any elements anyway, + # so we can treat it the same as non-iterables. + if obj is iter(obj): + return False + + # range reprs fine regardless of length. + if isinstance(obj, range): + return False + + # numpy and scipy collections (ndarray etc) have + # self-truncating repr, so they're always safe. + try: + module = type(obj).__module__.partition(".")[0] + if module in ("numpy", "scipy"): + return False + except Exception: + pass + + # Iterables that nest too deep are considered long. + if level >= len(self.maxcollection): + return True + + # It is too long if the length exceeds the limit, or any + # of its elements are long iterables. + if hasattr(obj, "__len__"): + try: + size = len(obj) + except Exception: + size = None + if size is not None and size > self.maxcollection[level]: + return True + return any(self._is_long_iter(item, level + 1) for item in obj) + return any( + i > self.maxcollection[level] or self._is_long_iter(item, level + 1) + for i, item in enumerate(obj) + ) + + except Exception: + # If anything breaks, assume the worst case. + return True + + def _repr_iter(self, obj, level, prefix, suffix, comma_after_single_element=False): # noqa: FBT002 + yield prefix + + if level >= len(self.maxcollection): + yield "..." + else: + count = self.maxcollection[level] + yield_comma = False + for item in obj: + if yield_comma: + yield ", " + yield_comma = True + + count -= 1 + if count <= 0: + yield "..." + break + + yield from self._repr(item, 100 if item is obj else level + 1) + else: + if comma_after_single_element: # noqa: SIM102 + if count == self.maxcollection[level] - 1: + yield "," + yield suffix + + def _repr_long_iter(self, obj): + try: + length = hex(len(obj)) if self.convert_to_hex else len(obj) + obj_repr = f"<{type(obj).__name__}, len() = {length}>" + except Exception: + try: + obj_repr = "<" + type(obj).__name__ + ">" + except Exception: + obj_repr = "" + yield obj_repr + + def _repr_dict(self, obj, level, prefix, suffix, item_prefix, item_sep, item_suffix): + if not obj: + yield prefix + suffix + return + if level >= len(self.maxcollection): + yield prefix + "..." + suffix + return + + yield prefix + + count = self.maxcollection[level] + yield_comma = False + + obj_keys = list(obj) + + for key in obj_keys: + if yield_comma: + yield ", " + yield_comma = True + + count -= 1 + if count <= 0: + yield "..." + break + + yield item_prefix + for p in self._repr(key, level + 1): + yield p + + yield item_sep + + try: + item = obj[key] + except Exception: + yield "" + else: + for p in self._repr(item, 100 if item is obj else level + 1): + yield p + yield item_suffix + + yield suffix + + def _repr_str(self, obj, level): + try: + if self.raw_value: + # For raw value retrieval, ignore all limits. + if isinstance(obj, bytes): + yield obj.decode("latin-1") + else: + yield obj + return + + limit_inner = self.maxother_inner + limit_outer = self.maxother_outer + limit = limit_inner if level > 0 else limit_outer + if len(obj) <= limit: + # Note that we check the limit before doing the repr (so, the final string + # may actually be considerably bigger on some cases, as besides + # the additional u, b, ' chars, some chars may be escaped in repr, so + # even a single char such as \U0010ffff may end up adding more + # chars than expected). + yield self._convert_to_unicode_or_bytes_repr(repr(obj)) + return + + # Slightly imprecise calculations - we may end up with a string that is + # up to 6 characters longer than limit. If you need precise formatting, + # you are using the wrong class. + left_count, right_count = max(1, int(2 * limit / 3)), max(1, int(limit / 3)) + + # Important: only do repr after slicing to avoid duplicating a byte array that could be + # huge. + + # Note: we don't deal with high surrogates here because we're not dealing with the + # repr() of a random object. + # i.e.: A high surrogate unicode char may be splitted on Py2, but as we do a `repr` + # afterwards, that's ok. + + # Also, we just show the unicode/string/bytes repr() directly to make clear what the + # input type was (so, on py2 a unicode would start with u' and on py3 a bytes would + # start with b'). + + part1 = obj[:left_count] + part1 = repr(part1) + part1 = part1[: part1.rindex("'")] # Remove the last ' + + part2 = obj[-right_count:] + part2 = repr(part2) + part2 = part2[part2.index("'") + 1 :] # Remove the first ' (and possibly u or b). + + yield part1 + yield "..." + yield part2 + except: # noqa: E722 + # This shouldn't really happen, but let's play it safe. + # exception('Error getting string representation to show.') + yield from self._repr_obj(obj, level, self.maxother_inner, self.maxother_outer) + + def _repr_other(self, obj, level): + return self._repr_obj(obj, level, self.maxother_inner, self.maxother_outer) + + def _repr_obj(self, obj, level, limit_inner, limit_outer): + try: + if self.raw_value: + # For raw value retrieval, ignore all limits. + if isinstance(obj, bytes): + yield obj.decode("latin-1") + return + + try: + mv = memoryview(obj) + except Exception: + yield self._convert_to_unicode_or_bytes_repr(repr(obj)) + return + else: + # Map bytes to Unicode codepoints with same values. + yield mv.tobytes().decode("latin-1") + return + elif self.convert_to_hex and isinstance(obj, self.int_types): + obj_repr = hex(obj) + else: + obj_repr = repr(obj) + except Exception: + try: + obj_repr = object.__repr__(obj) + except Exception: + try: + obj_repr = "" + except Exception: + obj_repr = "" + + limit = limit_inner if level > 0 else limit_outer + + if limit >= len(obj_repr): + yield self._convert_to_unicode_or_bytes_repr(obj_repr) + return + + # Slightly imprecise calculations - we may end up with a string that is + # up to 3 characters longer than limit. If you need precise formatting, + # you are using the wrong class. + left_count, right_count = max(1, int(2 * limit / 3)), max(1, int(limit / 3)) + + yield obj_repr[:left_count] + yield "..." + yield obj_repr[-right_count:] + + def _convert_to_unicode_or_bytes_repr(self, obj_repr): + return obj_repr + + def _bytes_as_unicode_if_possible(self, obj_repr): + # We try to decode with 3 possible encoding (sys.stdout.encoding, + # locale.getpreferredencoding() and 'utf-8). If no encoding can decode + # the input, we return the original bytes. + try_encodings = [] + encoding = self.sys_stdout_encoding or getattr(sys.stdout, "encoding", None) + if encoding: + try_encodings.append(encoding.lower()) + + preferred_encoding = self.locale_preferred_encoding or locale.getpreferredencoding() + if preferred_encoding: + preferred_encoding = preferred_encoding.lower() + if preferred_encoding not in try_encodings: + try_encodings.append(preferred_encoding) + + if "utf-8" not in try_encodings: + try_encodings.append("utf-8") + + for encoding in try_encodings: + try: + return obj_repr.decode(encoding) + except UnicodeDecodeError: # noqa: PERF203 + pass + + return obj_repr # Return the original version (in bytes) + + +class DisplayOptions: + def __init__(self, width, max_columns): + self.width = width + self.max_columns = max_columns + + +_safe_repr = SafeRepr() +_collection_types = ["list", "tuple", "set"] +_array_page_size = 50 + + +def _get_value(variable): + return _safe_repr(variable) + + +def _get_property_names(variable): + props = [] + private_props = [] + for prop in dir(variable): + if not prop.startswith("_"): + props.append(prop) + elif not prop.startswith("__"): + private_props.append(prop) + return props + private_props + + +def _get_full_type(var_type): + module = "" + if hasattr(var_type, "__module__") and var_type.__module__ != "builtins": + module = var_type.__module__ + "." + if hasattr(var_type, "__qualname__"): + return module + var_type.__qualname__ + elif hasattr(var_type, "__name__"): + return module + var_type.__name__ + return None + + +def _get_variable_description(variable): + result = {} + + var_type = type(variable) + result["type"] = _get_full_type(var_type) + if hasattr(var_type, "__mro__"): + result["interfaces"] = [_get_full_type(t) for t in var_type.__mro__] + + if hasattr(variable, "__len__") and result["type"] in _collection_types: + result["count"] = len(variable) + + result["hasNamedChildren"] = hasattr(variable, "__dict__") or isinstance(variable, dict) + + result["value"] = _get_value(variable) + return result + + +def _get_child_property(root, property_chain): + try: + variable = root + for prop in property_chain: + if isinstance(prop, int): + if hasattr(variable, "__getitem__"): + variable = variable[prop] + elif isinstance(variable, set): + variable = list(variable)[prop] + else: + return None + elif hasattr(variable, prop): + variable = getattr(variable, prop) + elif isinstance(variable, dict) and prop in variable: + variable = variable[prop] + else: + return None + except Exception: + return None + + return variable + + +types_to_exclude = ["module", "function", "method", "class", "type"] + + +### Get info on variables at the root level +def getVariableDescriptions(): # noqa: N802 + return [ + { + "name": varName, + **_get_variable_description(globals()[varName]), + "root": varName, + "propertyChain": [], + "language": "python", + } + for varName in globals() + if type(globals()[varName]).__name__ not in types_to_exclude + and not varName.startswith("__") + ] + + +### Get info on children of a variable reached through the given property chain +def getAllChildrenDescriptions(root_var_name, property_chain, start_index): # noqa: N802 + root = globals()[root_var_name] + if root is None: + return [] + + parent = root + if len(property_chain) > 0: + parent = _get_child_property(root, property_chain) + + children = [] + parent_info = _get_variable_description(parent) + if "count" in parent_info: + if parent_info["count"] > 0: + last_item = min(parent_info["count"], start_index + _array_page_size) + index_range = range(start_index, last_item) + children = [ + { + **_get_variable_description(_get_child_property(parent, [i])), + "name": str(i), + "root": root_var_name, + "propertyChain": [*property_chain, i], + "language": "python", + } + for i in index_range + ] + elif parent_info["hasNamedChildren"]: + children_names = [] + if hasattr(parent, "__dict__"): + children_names = _get_property_names(parent) + elif isinstance(parent, dict): + children_names = list(parent.keys()) + + children = [] + for prop in children_names: + child_property = _get_child_property(parent, [prop]) + if child_property is not None and type(child_property).__name__ not in types_to_exclude: + child = { + **_get_variable_description(child_property), + "name": prop, + "root": root_var_name, + "propertyChain": [*property_chain, prop], + } + children.append(child) + + return children diff --git a/python_files/tests/test_get_variable_info.py b/python_files/tests/test_get_variable_info.py new file mode 100644 index 000000000000..73f94fe26f06 --- /dev/null +++ b/python_files/tests/test_get_variable_info.py @@ -0,0 +1,114 @@ +import get_variable_info + + +def set_global_variable(value): + # setting on the module allows tests to set a variable that the module under test can access + get_variable_info.test_variable = value # pyright: ignore[reportGeneralTypeIssues] + + +def get_global_variable(): + results = get_variable_info.getVariableDescriptions() + for variable in results: + if variable["name"] == "test_variable": + return variable + return None + + +def assert_variable_found(variable, expected_value, expected_type, expected_count=None): + set_global_variable(variable) + variable = get_global_variable() + assert variable is not None + if expected_value is not None: + assert variable["value"] == expected_value + assert variable["type"] == expected_type + if expected_count is not None: + assert variable["count"] == expected_count + else: + assert "count" not in variable + return variable + + +def assert_indexed_child(variable, start_index, expected_index, expected_child_value=None): + children = get_variable_info.getAllChildrenDescriptions( + variable["root"], variable["propertyChain"], start_index + ) + child = children[expected_index] + + if expected_child_value is not None: + assert child["value"] == expected_child_value + return child + + +def assert_property(variable, expected_property_name, expected_property_value=None): + children = get_variable_info.getAllChildrenDescriptions( + variable["root"], variable["propertyChain"], 0 + ) + found = None + for child in children: + chain = child["propertyChain"] + property_name = chain[-1] if chain else None + if property_name == expected_property_name: + found = child + break + + assert found is not None + if expected_property_value is not None: + assert found["value"] == expected_property_value + return found + + +def test_simple(): + assert_variable_found(1, "1", "int", None) + + +def test_list(): + found = assert_variable_found([1, 2, 3], "[1, 2, 3]", "list", 3) + assert_indexed_child(found, 0, 0, "1") + + +def test_dict(): + found = assert_variable_found({"a": 1, "b": 2}, "{'a': 1, 'b': 2}", "dict", None) + assert found["hasNamedChildren"] + assert_property(found, "a", "1") + assert_property(found, "b", "2") + + +def test_tuple(): + found = assert_variable_found((1, 2, 3), "(1, 2, 3)", "tuple", 3) + assert_indexed_child(found, 0, 0, "1") + + +def test_set(): + found = assert_variable_found({1, 2, 3}, "{1, 2, 3}", "set", 3) + assert_indexed_child(found, 0, 0, "1") + + +def test_self_referencing_dict(): + d = {} + d["self"] = d + found = assert_variable_found(d, "{'self': {...}}", "dict", None) + assert_property(found, "self", "{'self': {...}}") + + +def test_nested_list(): + found = assert_variable_found([[1, 2], [3, 4]], "[[1, 2], [3, 4]]", "list", 2) + assert_indexed_child(found, 0, 0, "[1, 2]") + + +def test_long_list(): + child = assert_variable_found(list(range(1_000_000)), None, "list", 1_000_000) + value = child["value"] + assert value.startswith("[0, 1, 2, 3") + assert value.endswith("...]") + assert_indexed_child(child, 400_000, 10, "400010") + assert_indexed_child(child, 999_950, 10, "999960") + + +def test_get_nested_children(): + d = [{"a": {("hello")}}] + found = assert_variable_found(d, "[{'a': {...}}]", "list", 1) + + found = assert_indexed_child(found, 0, 0) + found = assert_property(found, "a") + found = assert_indexed_child(found, 0, 0) + assert found["value"] == "'hello'" diff --git a/src/client/repl/nativeRepl.ts b/src/client/repl/nativeRepl.ts index 8b233f765468..8e0337f8d276 100644 --- a/src/client/repl/nativeRepl.ts +++ b/src/client/repl/nativeRepl.ts @@ -19,6 +19,8 @@ import { executeNotebookCell, openInteractiveREPL, selectNotebookKernel } from ' import { createReplController } from './replController'; import { EventName } from '../telemetry/constants'; import { sendTelemetryEvent } from '../telemetry'; +import { VariablesProvider } from './variables/variablesProvider'; +import { VariableRequester } from './variables/variableRequester'; let nativeRepl: NativeRepl | undefined; // In multi REPL scenario, hashmap of URI to Repl. export class NativeRepl implements Disposable { @@ -48,7 +50,7 @@ export class NativeRepl implements Disposable { nativeRepl.interpreter = interpreter; await nativeRepl.setReplDirectory(); nativeRepl.pythonServer = createPythonServer([interpreter.path as string], nativeRepl.cwd); - nativeRepl.replController = nativeRepl.setReplController(); + nativeRepl.setReplController(); return nativeRepl; } @@ -111,7 +113,12 @@ export class NativeRepl implements Disposable { */ public setReplController(): NotebookController { if (!this.replController) { - return createReplController(this.interpreter!.path, this.disposables, this.cwd); + this.replController = createReplController(this.interpreter!.path, this.disposables, this.cwd); + this.replController.variableProvider = new VariablesProvider( + new VariableRequester(this.pythonServer), + () => this.notebookDocument, + this.pythonServer.onCodeExecuted, + ); } return this.replController; } diff --git a/src/client/repl/pythonServer.ts b/src/client/repl/pythonServer.ts index a342e989af7c..0f500f0431bc 100644 --- a/src/client/repl/pythonServer.ts +++ b/src/client/repl/pythonServer.ts @@ -1,7 +1,7 @@ import * as path from 'path'; import * as ch from 'child_process'; import * as rpc from 'vscode-jsonrpc/node'; -import { Disposable, window } from 'vscode'; +import { Disposable, Event, EventEmitter, window } from 'vscode'; import { EXTENSION_ROOT_DIR } from '../constants'; import { traceError, traceLog } from '../logging'; import { captureTelemetry } from '../telemetry'; @@ -15,15 +15,21 @@ export interface ExecutionResult { } export interface PythonServer extends Disposable { + onCodeExecuted: Event; execute(code: string): Promise; + executeSilently(code: string): Promise; interrupt(): void; input(): void; checkValidCommand(code: string): Promise; } -class PythonServerImpl implements Disposable { +class PythonServerImpl implements PythonServer, Disposable { private readonly disposables: Disposable[] = []; + private readonly _onCodeExecuted = new EventEmitter(); + + onCodeExecuted = this._onCodeExecuted.event; + constructor(private connection: rpc.MessageConnection, private pythonServer: ch.ChildProcess) { this.initialize(); this.input(); @@ -57,6 +63,18 @@ class PythonServerImpl implements Disposable { @captureTelemetry(EventName.EXECUTION_CODE, { scope: 'selection' }, false) public async execute(code: string): Promise { + const result = await this.executeCode(code); + if (result?.status) { + this._onCodeExecuted.fire(); + } + return result; + } + + public executeSilently(code: string): Promise { + return this.executeCode(code); + } + + private async executeCode(code: string): Promise { try { const result = await this.connection.sendRequest('execute', code); return result as ExecutionResult; diff --git a/src/client/repl/variables/types.ts b/src/client/repl/variables/types.ts new file mode 100644 index 000000000000..1e3c80d32077 --- /dev/null +++ b/src/client/repl/variables/types.ts @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CancellationToken, Variable } from 'vscode'; + +export interface IVariableDescription extends Variable { + /** The name of the variable at the root scope */ + root: string; + /** How to look up the specific property of the root variable */ + propertyChain: (string | number)[]; + /** The number of children for collection types */ + count?: number; + /** Names of children */ + hasNamedChildren?: boolean; + /** A method to get the children of this variable */ + getChildren?: (start: number, token: CancellationToken) => Promise; +} diff --git a/src/client/repl/variables/variableRequester.ts b/src/client/repl/variables/variableRequester.ts new file mode 100644 index 000000000000..e66afdcd6616 --- /dev/null +++ b/src/client/repl/variables/variableRequester.ts @@ -0,0 +1,59 @@ +import { CancellationToken } from 'vscode'; +import path from 'path'; +import * as fsapi from '../../common/platform/fs-paths'; +import { IVariableDescription } from './types'; +import { PythonServer } from '../pythonServer'; +import { EXTENSION_ROOT_DIR } from '../../constants'; + +const VARIABLE_SCRIPT_LOCATION = path.join(EXTENSION_ROOT_DIR, 'python_files', 'get_variable_info.py'); + +export class VariableRequester { + public static scriptContents: string | undefined; + + constructor(private pythonServer: PythonServer) {} + + async getAllVariableDescriptions( + parent: IVariableDescription | undefined, + start: number, + token: CancellationToken, + ): Promise { + const scriptLines = (await getContentsOfVariablesScript()).split(/(?:\r\n|\n)/); + if (parent) { + const printCall = `import json;return json.dumps(getAllChildrenDescriptions(\'${ + parent.root + }\', ${JSON.stringify(parent.propertyChain)}, ${start}))`; + scriptLines.push(printCall); + } else { + scriptLines.push('import json;return json.dumps(getVariableDescriptions())'); + } + + if (token.isCancellationRequested) { + return []; + } + + const script = wrapScriptInFunction(scriptLines); + const result = await this.pythonServer.executeSilently(script); + + if (result?.output && !token.isCancellationRequested) { + return JSON.parse(result.output) as IVariableDescription[]; + } + + return []; + } +} + +function wrapScriptInFunction(scriptLines: string[]): string { + const indented = scriptLines.map((line) => ` ${line}`).join('\n'); + // put everything into a function scope and then delete that scope + // TODO: run in a background thread + return `def __VSCODE_run_script():\n${indented}\nprint(__VSCODE_run_script())\ndel __VSCODE_run_script`; +} + +async function getContentsOfVariablesScript(): Promise { + if (VariableRequester.scriptContents) { + return VariableRequester.scriptContents; + } + const contents = await fsapi.readFile(VARIABLE_SCRIPT_LOCATION, 'utf-8'); + VariableRequester.scriptContents = contents; + return VariableRequester.scriptContents; +} diff --git a/src/client/repl/variables/variableResultCache.ts b/src/client/repl/variables/variableResultCache.ts new file mode 100644 index 000000000000..1e19415becb7 --- /dev/null +++ b/src/client/repl/variables/variableResultCache.ts @@ -0,0 +1,28 @@ +import { VariablesResult } from 'vscode'; + +export class VariableResultCache { + private cache = new Map(); + + private executionCount = 0; + + getResults(executionCount: number, cacheKey: string): VariablesResult[] | undefined { + if (this.executionCount !== executionCount) { + this.cache.clear(); + this.executionCount = executionCount; + } + + return this.cache.get(cacheKey); + } + + setResults(executionCount: number, cacheKey: string, results: VariablesResult[]): void { + if (this.executionCount < executionCount) { + this.cache.clear(); + this.executionCount = executionCount; + } else if (this.executionCount > executionCount) { + // old results, don't cache + return; + } + + this.cache.set(cacheKey, results); + } +} diff --git a/src/client/repl/variables/variablesProvider.ts b/src/client/repl/variables/variablesProvider.ts new file mode 100644 index 000000000000..ffb7c221d00c --- /dev/null +++ b/src/client/repl/variables/variablesProvider.ts @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + CancellationToken, + NotebookDocument, + Variable, + NotebookVariablesRequestKind, + VariablesResult, + EventEmitter, + Event, + NotebookVariableProvider, +} from 'vscode'; +import { VariableResultCache } from './variableResultCache'; +import { IVariableDescription } from './types'; +import { VariableRequester } from './variableRequester'; + +export class VariablesProvider implements NotebookVariableProvider { + private readonly variableResultCache = new VariableResultCache(); + + private _onDidChangeVariables = new EventEmitter(); + + onDidChangeVariables = this._onDidChangeVariables.event; + + private executionCount = 0; + + constructor( + private readonly variableRequester: VariableRequester, + private readonly getNotebookDocument: () => NotebookDocument | undefined, + codeExecutedEvent: Event, + ) { + codeExecutedEvent(() => this.onDidExecuteCode()); + } + + onDidExecuteCode(): void { + const notebook = this.getNotebookDocument(); + if (notebook) { + this.executionCount += 1; + this._onDidChangeVariables.fire(notebook); + } + } + + async *provideVariables( + notebook: NotebookDocument, + parent: Variable | undefined, + kind: NotebookVariablesRequestKind, + start: number, + token: CancellationToken, + ): AsyncIterable { + const notebookDocument = this.getNotebookDocument(); + if (token.isCancellationRequested || !notebookDocument || notebookDocument !== notebook) { + return; + } + + const { executionCount } = this; + const cacheKey = getVariableResultCacheKey(notebook.uri.toString(), parent, start); + let results = this.variableResultCache.getResults(executionCount, cacheKey); + + if (parent) { + const parentDescription = parent as IVariableDescription; + if (!results && parentDescription.getChildren) { + const variables = await parentDescription.getChildren(start, token); + if (token.isCancellationRequested) { + return; + } + results = variables.map((variable) => this.createVariableResult(variable)); + this.variableResultCache.setResults(executionCount, cacheKey, results); + } else if (!results) { + // no cached results and no way to get children, so return empty + return; + } + + for (const result of results) { + yield result; + } + + // check if we have more indexed children to return + if ( + kind === 2 && + parentDescription.count && + results.length > 0 && + parentDescription.count > start + results.length + ) { + for await (const result of this.provideVariables( + notebook, + parent, + kind, + start + results.length, + token, + )) { + yield result; + } + } + } else { + if (!results) { + const variables = await this.variableRequester.getAllVariableDescriptions(undefined, start, token); + if (token.isCancellationRequested) { + return; + } + results = variables.map((variable) => this.createVariableResult(variable)); + this.variableResultCache.setResults(executionCount, cacheKey, results); + } + + for (const result of results) { + yield result; + } + } + } + + private createVariableResult(result: IVariableDescription): VariablesResult { + const indexedChildrenCount = result.count ?? 0; + const hasNamedChildren = !!result.hasNamedChildren; + const variable = { + getChildren: (start: number, token: CancellationToken) => this.getChildren(variable, start, token), + expression: createExpression(result.root, result.propertyChain), + ...result, + } as Variable; + return { variable, hasNamedChildren, indexedChildrenCount }; + } + + async getChildren(variable: Variable, start: number, token: CancellationToken): Promise { + const parent = variable as IVariableDescription; + return this.variableRequester.getAllVariableDescriptions(parent, start, token); + } +} + +function createExpression(root: string, propertyChain: (string | number)[]): string { + let expression = root; + for (const property of propertyChain) { + if (typeof property === 'string') { + expression += `.${property}`; + } else { + expression += `[${property}]`; + } + } + return expression; +} + +function getVariableResultCacheKey(uri: string, parent: Variable | undefined, start: number) { + let parentKey = ''; + const parentDescription = parent as IVariableDescription; + if (parentDescription) { + parentKey = `${parentDescription.name}.${parentDescription.propertyChain.join('.')}[[${start}`; + } + return `${uri}:${parentKey}`; +} diff --git a/src/test/repl/variableProvider.test.ts b/src/test/repl/variableProvider.test.ts new file mode 100644 index 000000000000..8b45fae0c5a0 --- /dev/null +++ b/src/test/repl/variableProvider.test.ts @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { assert } from 'chai'; +import { NotebookDocument, CancellationTokenSource, VariablesResult, Variable, EventEmitter } from 'vscode'; +import * as TypeMoq from 'typemoq'; +import { IVariableDescription } from '../../client/repl/variables/types'; +import { VariablesProvider } from '../../client/repl/variables/variablesProvider'; +import { VariableRequester } from '../../client/repl/variables/variableRequester'; + +suite.only('ReplVariablesProvider', () => { + let provider: VariablesProvider; + let varRequester: TypeMoq.IMock; + let notebook: TypeMoq.IMock; + const executionEventEmitter = new EventEmitter(); + const cancellationToken = new CancellationTokenSource().token; + + const objectVariable: IVariableDescription = { + name: 'myObject', + value: '...', + root: 'myObject', + hasNamedChildren: true, + propertyChain: [], + }; + + const listVariable: IVariableDescription = { + name: 'myList', + value: '[...]', + count: 3, + root: 'myObject', + propertyChain: ['myList'], + }; + + function createListItem(index: number): IVariableDescription { + return { + name: index.toString(), + value: `value${index}`, + count: index, + root: 'myObject', + propertyChain: ['myList', index], + }; + } + + function setVariablesForParent( + parent: IVariableDescription | undefined, + result: IVariableDescription[], + updated?: IVariableDescription[], + startIndex?: number, + ) { + let returnedOnce = false; + varRequester + .setup((v) => v.getAllVariableDescriptions(parent, startIndex ?? TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns(() => { + if (updated && returnedOnce) { + return Promise.resolve(updated); + } + returnedOnce = true; + return Promise.resolve(result); + }); + } + + async function provideVariables(parent: Variable | undefined, kind = 1) { + const results: VariablesResult[] = []; + for await (const result of provider.provideVariables(notebook.object, parent, kind, 0, cancellationToken)) { + results.push(result); + } + return results; + } + + setup(() => { + varRequester = TypeMoq.Mock.ofType(); + notebook = TypeMoq.Mock.ofType(); + provider = new VariablesProvider(varRequester.object, () => notebook.object, executionEventEmitter.event); + }); + + test('provideVariables without parent should yield variables', async () => { + setVariablesForParent(undefined, [objectVariable]); + + const results = await provideVariables(undefined); + + assert.isNotEmpty(results); + assert.equal(results.length, 1); + assert.equal(results[0].variable.name, 'myObject'); + assert.equal(results[0].variable.expression, 'myObject'); + }); + + test('provideVariables with a parent should call get children correctly', async () => { + const listVariableItems = [0, 1, 2].map(createListItem); + setVariablesForParent(undefined, [objectVariable]); + + // pass each the result as the parent in the next call + const rootVariable = (await provideVariables(undefined))[0]; + setVariablesForParent(rootVariable.variable as IVariableDescription, [listVariable]); + const listResult = (await provideVariables(rootVariable!.variable))[0]; + setVariablesForParent(listResult.variable as IVariableDescription, listVariableItems); + const listItems = await provideVariables(listResult!.variable, 2); + + assert.equal(listResult.variable.name, 'myList'); + assert.equal(listResult.variable.expression, 'myObject.myList'); + assert.isNotEmpty(listItems); + assert.equal(listItems.length, 3); + listItems.forEach((item, index) => { + assert.equal(item.variable.name, index.toString()); + assert.equal(item.variable.value, `value${index}`); + assert.equal(item.variable.expression, `myObject.myList[${index}]`); + }); + }); + + test('All indexed variables should be returned when requested', async () => { + const listVariable: IVariableDescription = { + name: 'myList', + value: '[...]', + count: 6, + root: 'myList', + propertyChain: [], + }; + + setVariablesForParent(undefined, [listVariable]); + const rootVariable = (await provideVariables(undefined))[0]; + const firstPage = [0, 1, 2].map(createListItem); + const secondPage = [3, 4, 5].map(createListItem); + setVariablesForParent(rootVariable.variable as IVariableDescription, firstPage, undefined, 0); + setVariablesForParent(rootVariable.variable as IVariableDescription, secondPage, undefined, firstPage.length); + + const listItemResult = await provideVariables(rootVariable!.variable, 2); + + assert.equal(listItemResult.length, 6, 'full list of items should be returned'); + listItemResult.forEach((item, index) => { + assert.equal(item.variable.name, index.toString()); + assert.equal(item.variable.value, `value${index}`); + }); + }); + + test('Getting less indexed items than the specified count is handled', async () => { + const listVariable: IVariableDescription = { + name: 'myList', + value: '[...]', + count: 6, + root: 'myList', + propertyChain: [], + }; + + const firstPage = [0, 1, 2].map(createListItem); + const secondPage = [3, 4].map(createListItem); + setVariablesForParent(undefined, [listVariable]); + const rootVariable = (await provideVariables(undefined))[0]; + setVariablesForParent(rootVariable.variable as IVariableDescription, firstPage, undefined, 0); + setVariablesForParent(rootVariable.variable as IVariableDescription, secondPage, undefined, firstPage.length); + setVariablesForParent(rootVariable.variable as IVariableDescription, [], undefined, 5); + + const listItemResult = await provideVariables(rootVariable!.variable, 2); + + assert.equal(listItemResult.length, 5); + listItemResult.forEach((item, index) => { + assert.equal(item.variable.name, index.toString()); + assert.equal(item.variable.value, `value${index}`); + }); + }); + + test('Getting variables again with new execution count should get updated variables', async () => { + const intVariable: IVariableDescription = { + name: 'myInt', + value: '1', + root: '', + propertyChain: [], + }; + setVariablesForParent(undefined, [intVariable], [{ ...intVariable, value: '2' }]); + + const first = await provideVariables(undefined); + executionEventEmitter.fire(); + const second = await provideVariables(undefined); + + assert.equal(first.length, 1); + assert.equal(second.length, 1); + assert.equal(first[0].variable.value, '1'); + assert.equal(second[0].variable.value, '2'); + }); + + test('Getting variables again with same execution count should not make another call', async () => { + const intVariable: IVariableDescription = { + name: 'myInt', + value: '1', + root: '', + propertyChain: [], + }; + + setVariablesForParent(undefined, [intVariable]); + + const first = await provideVariables(undefined); + const second = await provideVariables(undefined); + + assert.equal(first.length, 1); + assert.equal(second.length, 1); + assert.equal(first[0].variable.value, '1'); + + varRequester.verify( + (x) => x.getAllVariableDescriptions(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), + TypeMoq.Times.once(), + ); + }); + + test('Cache pages of indexed children correctly', async () => { + const listVariable: IVariableDescription = { + name: 'myList', + value: '[...]', + count: 6, + root: 'myList', + propertyChain: [], + }; + + const firstPage = [0, 1, 2].map(createListItem); + const secondPage = [3, 4, 5].map(createListItem); + setVariablesForParent(undefined, [listVariable]); + const rootVariable = (await provideVariables(undefined))[0]; + setVariablesForParent(rootVariable.variable as IVariableDescription, firstPage, undefined, 0); + setVariablesForParent(rootVariable.variable as IVariableDescription, secondPage, undefined, firstPage.length); + + await provideVariables(rootVariable!.variable, 2); + + // once for the parent and once for each of the two pages of list items + varRequester.verify( + (x) => x.getAllVariableDescriptions(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), + TypeMoq.Times.exactly(3), + ); + + const listItemResult = await provideVariables(rootVariable!.variable, 2); + + assert.equal(listItemResult.length, 6, 'full list of items should be returned'); + listItemResult.forEach((item, index) => { + assert.equal(item.variable.name, index.toString()); + assert.equal(item.variable.value, `value${index}`); + }); + + // no extra calls for getting the children again + varRequester.verify( + (x) => x.getAllVariableDescriptions(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()), + TypeMoq.Times.exactly(3), + ); + }); +}); diff --git a/types/vscode.proposed.notebookVariableProvider.d.ts b/types/vscode.proposed.notebookVariableProvider.d.ts new file mode 100644 index 000000000000..4fac96c45f0a --- /dev/null +++ b/types/vscode.proposed.notebookVariableProvider.d.ts @@ -0,0 +1,55 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +declare module 'vscode' { + + export interface NotebookController { + /** Set this to attach a variable provider to this controller. */ + variableProvider?: NotebookVariableProvider; + } + + export enum NotebookVariablesRequestKind { + Named = 1, + Indexed = 2 + } + + interface VariablesResult { + variable: Variable; + hasNamedChildren: boolean; + indexedChildrenCount: number; + } + + interface NotebookVariableProvider { + onDidChangeVariables: Event; + + /** When parent is undefined, this is requesting global Variables. When a variable is passed, it's requesting child props of that Variable. */ + provideVariables(notebook: NotebookDocument, parent: Variable | undefined, kind: NotebookVariablesRequestKind, start: number, token: CancellationToken): AsyncIterable; + } + + interface Variable { + /** The variable's name. */ + name: string; + + /** The variable's value. + This can be a multi-line text, e.g. for a function the body of a function. + For structured variables (which do not have a simple value), it is recommended to provide a one-line representation of the structured object. + This helps to identify the structured object in the collapsed state when its children are not yet visible. + An empty string can be used if no value should be shown in the UI. + */ + value: string; + + /** The code that represents how the variable would be accessed in the runtime environment */ + expression?: string; + + /** The type of the variable's value */ + type?: string; + + /** The interfaces or contracts that the type satisfies */ + interfaces?: string[]; + + /** The language of the variable's value */ + language?: string; + } + +}