Formatting on Enter (#649)

* Basic tokenizer

* Fixed property names

* Tests, round I

* Tests, round II

* tokenizer test

* Remove temorary change

* Fix merge issue

* Merge conflict

* Merge conflict

* Completion test

* Fix last line

* Fix javascript math

* Make test await for results

* Add license headers

* Rename definitions to types

* License headers

* Fix typo in completion details (typo)

* Fix hover test

* Russian translations

* Update to better translation

* Fix typo

*  #70 How to get all parameter info when filling in a function param list

* Fix #70 How to get all parameter info when filling in a function param list

* Clean up

* Clean imports

* CR feedback

* Trim whitespace for test stability

* More tests

* Better handle no-parameters documentation

* Better handle ellipsis and Python3

* #385 Auto-Indentation doesn't work after comment

* #141 Auto indentation broken when return keyword involved

* Undo changes

* On type formatting

* Fix warnings

* Round I

* Round 2

* Round 3

* Round 4

* Round 5

* no message

* Round 6

* Round 7

* Clean up targets and messages

* Settings propagation

* Tests

* Test warning

* Fix installer tests

* Tests

* Test fixes

* Fix terminal service and tests async/await

* Fix mock setup

* Test fix

* Test async/await fix

* Test fix + activate tslint on awaits

* Use command manager

* Work around updateSettings

* Multiroot fixes, partial

* More workarounds

* Multiroot tests

* Fix installer test

* Test fixes

* Disable prospector

* Enable dispose in all cases

* Fix event firing

* Min pylint options

* Min checkers & pylintrc discovery

* Fix Windows path in tests for Travis

* Fix Mac test

* Test fix

* Work around VSC issue with formatting on save #624

* Workaround test

* Unused

* Old file

* Brace and colon handling

* More tests

* Don't format inside strings and comments

* Provider tests

* Remove duplicate code

package.json

@@ -1594,6 +1594,7 @@
 		"tree-kill": "^1.1.0",
 		"typescript-char": "^0.0.0",
 		"uint64be": "^1.0.1",
+		"unicode": "^10.0.0",
 		"untildify": "^3.0.2",
 		"vscode-debugadapter": "^1.0.1",
 		"vscode-debugprotocol": "^1.0.1",

src/client/extension.ts

@@ -57,6 +57,7 @@ import { StopWatch } from './telemetry/stopWatch';
 import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry';
 import { ICodeExecutionManager } from './terminals/types';
 import { BlockFormatProviders } from './typeFormatters/blockFormatProvider';
+import { OnEnterFormatter } from './typeFormatters/onEnterFormatter';
 import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants';
 import * as tests from './unittests/main';
 import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry';
@@ -190,6 +191,8 @@ export async function activate(context: vscode.ExtensionContext) {
     context.subscriptions.push(new WorkspaceSymbols(serviceContainer));
 
     context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
+    context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
+
     // In case we have CR LF
     const triggerCharacters: string[] = os.EOL.split('');
     triggerCharacters.shift();

src/client/formatters/lineFormatter.ts

@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable-next-line:import-name
+import Char from 'typescript-char';
+import { BraceCounter } from '../language/braceCounter';
+import { TextBuilder } from '../language/textBuilder';
+import { Tokenizer } from '../language/tokenizer';
+import { ITextRangeCollection, IToken, TokenType } from '../language/types';
+
+export class LineFormatter {
+    private builder: TextBuilder;
+    private tokens: ITextRangeCollection<IToken>;
+    private braceCounter: BraceCounter;
+    private text: string;
+
+    // tslint:disable-next-line:cyclomatic-complexity
+    public formatLine(text: string): string {
+        this.tokens = new Tokenizer().tokenize(text);
+        this.text = text;
+        this.builder = new TextBuilder();
+        this.braceCounter = new BraceCounter();
+
+        if (this.tokens.count === 0) {
+            return this.text;
+        }
+
+        const ws = this.text.substr(0, this.tokens.getItemAt(0).start);
+        if (ws.length > 0) {
+            this.builder.append(ws); // Preserve leading indentation
+        }
+
+        for (let i = 0; i < this.tokens.count; i += 1) {
+            const t = this.tokens.getItemAt(i);
+            const prev = i > 0 ? this.tokens.getItemAt(i - 1) : undefined;
+            const next = i < this.tokens.count - 1 ? this.tokens.getItemAt(i + 1) : undefined;
+
+            switch (t.type) {
+                case TokenType.Operator:
+                    this.handleOperator(i);
+                    break;
+
+                case TokenType.Comma:
+                    this.builder.append(',');
+                    if (next && !this.isCloseBraceType(next.type)) {
+                        this.builder.softAppendSpace();
+                    }
+                    break;
+
+                case TokenType.Identifier:
+                    if (!prev || (!this.isOpenBraceType(prev.type) && prev.type !== TokenType.Colon)) {
+                        this.builder.softAppendSpace();
+                    }
+                    this.builder.append(this.text.substring(t.start, t.end));
+                    break;
+
+                case TokenType.Colon:
+                    // x: 1 if not in slice, x[1:y] if inside the slice
+                    this.builder.append(':');
+                    if (!this.braceCounter.isOpened(TokenType.OpenBracket) && (next && next.type !== TokenType.Colon)) {
+                        // Not inside opened [[ ... ] sequence
+                        this.builder.softAppendSpace();
+                    }
+                    break;
+
+                case TokenType.Comment:
+                    // add space before in-line comment
+                    if (prev) {
+                        this.builder.softAppendSpace();
+                    }
+                    this.builder.append(this.text.substring(t.start, t.end));
+                    break;
+
+                default:
+                    this.handleOther(t);
+                    break;
+            }
+        }
+        return this.builder.getText();
+    }
+
+    private handleOperator(index: number): void {
+        const t = this.tokens.getItemAt(index);
+        if (index >= 2 && t.length === 1 && this.text.charCodeAt(t.start) === Char.Equal) {
+            if (this.braceCounter.isOpened(TokenType.OpenBrace)) {
+                // Check if this is = in function arguments. If so, do not
+                // add spaces around it.
+                const prev = this.tokens.getItemAt(index - 1);
+                const prevPrev = this.tokens.getItemAt(index - 2);
+                if (prev.type === TokenType.Identifier &&
+                    (prevPrev.type === TokenType.Comma || prevPrev.type === TokenType.OpenBrace)) {
+                    this.builder.append('=');
+                    return;
+                }
+            }
+        }
+        this.builder.softAppendSpace();
+        this.builder.append(this.text.substring(t.start, t.end));
+        this.builder.softAppendSpace();
+    }
+
+    private handleOther(t: IToken): void {
+        if (this.isBraceType(t.type)) {
+            this.braceCounter.countBrace(t);
+        }
+        this.builder.append(this.text.substring(t.start, t.end));
+    }
+
+    private isOpenBraceType(type: TokenType): boolean {
+        return type === TokenType.OpenBrace || type === TokenType.OpenBracket || type === TokenType.OpenCurly;
+    }
+    private isCloseBraceType(type: TokenType): boolean {
+        return type === TokenType.CloseBrace || type === TokenType.CloseBracket || type === TokenType.CloseCurly;
+    }
+    private isBraceType(type: TokenType): boolean {
+        return this.isOpenBraceType(type) || this.isCloseBraceType(type);
+    }
+}

src/client/language/braceCounter.ts

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { IToken, TokenType } from './types';
+
+class BracePair {
+    public readonly openBrace: TokenType;
+    public readonly closeBrace: TokenType;
+
+    constructor(openBrace: TokenType, closeBrace: TokenType) {
+        this.openBrace = openBrace;
+        this.closeBrace = closeBrace;
+    }
+}
+
+class Stack {
+    private store: IToken[] = [];
+    public push(val: IToken) {
+        this.store.push(val);
+    }
+    public pop(): IToken | undefined {
+        return this.store.pop();
+    }
+    public get length(): number {
+        return this.store.length;
+    }
+}
+
+export class BraceCounter {
+    private readonly bracePairs: BracePair[] = [
+        new BracePair(TokenType.OpenBrace, TokenType.CloseBrace),
+        new BracePair(TokenType.OpenBracket, TokenType.CloseBracket),
+        new BracePair(TokenType.OpenCurly, TokenType.CloseCurly)
+    ];
+    private braceStacks: Stack[] = [new Stack(), new Stack(), new Stack()];
+
+    public get count(): number {
+        let c = 0;
+        for (const s of this.braceStacks) {
+            c += s.length;
+        }
+        return c;
+    }
+
+    public isOpened(type: TokenType): boolean {
+        for (let i = 0; i < this.bracePairs.length; i += 1) {
+            const pair = this.bracePairs[i];
+            if (pair.openBrace === type || pair.closeBrace === type) {
+                return this.braceStacks[i].length > 0;
+            }
+        }
+        return false;
+    }
+
+    public countBrace(brace: IToken): boolean {
+        for (let i = 0; i < this.bracePairs.length; i += 1) {
+            const pair = this.bracePairs[i];
+            if (pair.openBrace === brace.type) {
+                this.braceStacks[i].push(brace);
+                return true;
+            }
+            if (pair.closeBrace === brace.type) {
+                if (this.braceStacks[i].length > 0) {
+                    this.braceStacks[i].pop();
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+}

src/client/language/characterStream.ts

@@ -4,6 +4,7 @@
 
 // tslint:disable-next-line:import-name
 import Char from 'typescript-char';
+import { isLineBreak, isWhiteSpace } from './characters';
 import { TextIterator } from './textIterator';
 import { ICharacterStream, ITextIterator } from './types';
 
@@ -70,11 +71,11 @@ export class CharacterStream implements ICharacterStream {
     }
 
     public isAtWhiteSpace(): boolean {
-        return this.currentChar <= Char.Space || this.currentChar === 0x200B; // Unicode whitespace
+        return isWhiteSpace(this.currentChar);
     }
 
     public isAtLineBreak(): boolean {
-        return this.currentChar === Char.CarriageReturn || this.currentChar === Char.LineFeed;
+        return isLineBreak(this.currentChar);
     }
 
     public skipLineBreak(): void {

src/client/language/characters.ts

@@ -0,0 +1,100 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable-next-line:import-name
+import Char from 'typescript-char';
+import { getUnicodeCategory, UnicodeCategory } from './unicode';
+
+export function isIdentifierStartChar(ch: number) {
+    switch (ch) {
+        // Underscore is explicitly allowed to start an identifier
+        case Char.Underscore:
+            return true;
+        // Characters with the Other_ID_Start property
+        case 0x1885:
+        case 0x1886:
+        case 0x2118:
+        case 0x212E:
+        case 0x309B:
+        case 0x309C:
+            return true;
+        default:
+            break;
+    }
+
+    const cat = getUnicodeCategory(ch);
+    switch (cat) {
+        // Supported categories for starting an identifier
+        case UnicodeCategory.UppercaseLetter:
+        case UnicodeCategory.LowercaseLetter:
+        case UnicodeCategory.TitlecaseLetter:
+        case UnicodeCategory.ModifierLetter:
+        case UnicodeCategory.OtherLetter:
+        case UnicodeCategory.LetterNumber:
+            return true;
+        default:
+            break;
+    }
+    return false;
+}
+
+export function isIdentifierChar(ch: number) {
+    if (isIdentifierStartChar(ch)) {
+        return true;
+    }
+
+    switch (ch) {
+        // Characters with the Other_ID_Continue property
+        case 0x00B7:
+        case 0x0387:
+        case 0x1369:
+        case 0x136A:
+        case 0x136B:
+        case 0x136C:
+        case 0x136D:
+        case 0x136E:
+        case 0x136F:
+        case 0x1370:
+        case 0x1371:
+        case 0x19DA:
+            return true;
+        default:
+            break;
+    }
+
+    switch (getUnicodeCategory(ch)) {
+        // Supported categories for continuing an identifier
+        case UnicodeCategory.NonSpacingMark:
+        case UnicodeCategory.SpacingCombiningMark:
+        case UnicodeCategory.DecimalDigitNumber:
+        case UnicodeCategory.ConnectorPunctuation:
+            return true;
+        default:
+            break;
+    }
+    return false;
+}
+
+export function isWhiteSpace(ch: number): boolean {
+    return ch <= Char.Space || ch === 0x200B; // Unicode whitespace
+}
+
+export function isLineBreak(ch: number): boolean {
+    return ch === Char.CarriageReturn || ch === Char.LineFeed;
+}
+
+export function isDecimal(ch: number): boolean {
+    return ch >= Char._0 && ch <= Char._9;
+}
+
+export function isHex(ch: number): boolean {
+    return isDecimal(ch) || (ch >= Char.a && ch <= Char.f) || (ch >= Char.A && ch <= Char.F);
+}
+
+export function isOctal(ch: number): boolean {
+    return ch >= Char._0 && ch <= Char._7;
+}
+
+export function isBinary(ch: number): boolean {
+    return ch === Char._0 || ch === Char._1;
+}

src/client/language/textBuilder.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { isWhiteSpace } from './characters';
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+export class TextBuilder {
+    private segments: string[] = [];
+
+    public getText(): string {
+        if (this.isLastWhiteSpace()) {
+            this.segments.pop();
+        }
+        return this.segments.join('');
+    }
+
+    public softAppendSpace(): void {
+        if (!this.isLastWhiteSpace() && this.segments.length > 0) {
+            this.segments.push(' ');
+        }
+    }
+
+    public append(text: string): void {
+        this.segments.push(text);
+    }
+
+    private isLastWhiteSpace(): boolean {
+        return this.segments.length > 0 && this.isWhitespace(this.segments[this.segments.length - 1]);
+    }
+
+    private isWhitespace(s: string): boolean {
+        for (let i = 0; i < s.length; i += 1) {
+            if (!isWhiteSpace(s.charCodeAt(i))) {
+                return false;
+            }
+        }
+        return true;
+    }
+}