From d0acc92006d54c167806c344fca08463c2856d94 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 00:22:12 +0100 Subject: [PATCH 1/6] multiline strings inside interpolations --- include/swift/AST/DiagnosticsParse.def | 2 -- include/swift/Parse/Lexer.h | 3 --- lib/Parse/Lexer.cpp | 28 ++++++++++++++++---------- test/Parse/multiline_errors.swift | 7 ------- test/Parse/multiline_string.swift | 25 +++++++++++++++++++++++ 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/include/swift/AST/DiagnosticsParse.def b/include/swift/AST/DiagnosticsParse.def index e0a8388152f46..3f366ff6056e0 100644 --- a/include/swift/AST/DiagnosticsParse.def +++ b/include/swift/AST/DiagnosticsParse.def @@ -134,8 +134,6 @@ ERROR(lex_illegal_multiline_string_end,none, "invalid end of multi-line string literal", ()) ERROR(lex_ambiguous_string_indent,none, "invalid mix of multi-line string literal indentation", ()) -ERROR(lex_multiline_inside_interpolation,none, - "multi-line string literals are not allowed inside an interpolation", ()) ERROR(lex_invalid_character,none, diff --git a/include/swift/Parse/Lexer.h b/include/swift/Parse/Lexer.h index c5dd2da1e9868..a56af6680785b 100644 --- a/include/swift/Parse/Lexer.h +++ b/include/swift/Parse/Lexer.h @@ -519,9 +519,6 @@ class Lexer { /// Try to lex conflict markers by checking for the presence of the start and /// end of the marker in diff3 or Perforce style respectively. bool tryLexConflictMarker(); - - /// Check multiline string literal is indented correctly. - void validateMultilineIndents(const Token &Str); }; } // end namespace swift diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp index 2579cdd9d1180..794a66e8f9a0c 100644 --- a/lib/Parse/Lexer.cpp +++ b/lib/Parse/Lexer.cpp @@ -1240,7 +1240,8 @@ unsigned Lexer::lexCharacter(const char *&CurPtr, char StopQuote, /// outstanding delimiters as it scans the string. static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, const char *EndPtr, - DiagnosticEngine *Diags) { + DiagnosticEngine *Diags, + bool MultilineString) { llvm::SmallVector OpenDelimiters; auto inStringLiteral = [&]() { return !OpenDelimiters.empty() && @@ -1257,9 +1258,11 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, // issues with malformed tokens or other problems. switch (*CurPtr++) { // String literals in general cannot be split across multiple lines; - // interpolated ones are no exception. + // interpolated ones are no exception (unless multiline strings.) case '\n': case '\r': + if (MultilineString) + continue; // Will be diagnosed as an unterminated string literal. return CurPtr-1; @@ -1272,12 +1275,12 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, } // Otherwise it's an ordinary character; treat it normally. } else { - if (*CurPtr == '"' && *(CurPtr + 1) == '"') { - Diags->diagnose(Lexer::getSourceLoc(CurPtr-1), - diag::lex_multiline_inside_interpolation); - } OpenDelimiters.push_back(CurPtr[-1]); } + if (*CurPtr == '"' && *(CurPtr + 1) == '"' && *(CurPtr - 1) == '"') { + MultilineString = true; + CurPtr += 2; + } continue; case '\\': if (inStringLiteral()) { @@ -1373,7 +1376,8 @@ static StringRef getMultilineTrailingIndent(const Token &Str, /// validateMultilineIndents: /// Diagnose contents of string literal that have inconsistent indentation. -void Lexer::validateMultilineIndents(const Token &Str) { +static void validateMultilineIndents(const Token &Str, + DiagnosticEngine *Diags) { StringRef Indent = getMultilineTrailingIndent(Str, Diags); if (Indent.empty()) return; @@ -1385,7 +1389,8 @@ void Lexer::validateMultilineIndents(const Token &Str) { size_t nextpos = pos + 1; if (BytesPtr[nextpos] != '\n' && BytesPtr[nextpos] != '\r') { if (Bytes.substr(nextpos, Indent.size()) != Indent) - diagnose(BytesPtr + nextpos, diag::lex_ambiguous_string_indent); + Diags->diagnose(Lexer::getSourceLoc(BytesPtr + nextpos), + diag::lex_ambiguous_string_indent); } pos = nextpos; } @@ -1415,7 +1420,8 @@ void Lexer::lexStringLiteral() { // Consume tokens until we hit the corresponding ')'. CurPtr += 2; const char *EndPtr = - skipToEndOfInterpolatedExpression(CurPtr, BufferEnd, Diags); + skipToEndOfInterpolatedExpression(CurPtr, BufferEnd, + Diags, MultilineString); if (*EndPtr == ')') { // Successfully scanned the body of the expression literal. @@ -1483,7 +1489,7 @@ void Lexer::lexStringLiteral() { if (*CurPtr == '"' && *(CurPtr + 1) == '"' && *(CurPtr + 2) != '"') { CurPtr += 2; formToken(tok::string_literal, TokStart, MultilineString); - validateMultilineIndents(NextToken); + validateMultilineIndents(NextToken, Diags); return; } else @@ -1778,7 +1784,7 @@ void Lexer::getStringLiteralSegments( // Find the closing ')'. const char *End = skipToEndOfInterpolatedExpression(BytesPtr, Str.getText().end(), - Diags); + Diags, MultilineString); assert(*End == ')' && "invalid string literal interpolations should" " not be returned as string literals"); ++End; diff --git a/test/Parse/multiline_errors.swift b/test/Parse/multiline_errors.swift index e32825a243153..5a478c82ec4fa 100644 --- a/test/Parse/multiline_errors.swift +++ b/test/Parse/multiline_errors.swift @@ -49,10 +49,3 @@ _ = """""" // expected-error@-0{{invalid start of multi-line string literal}} _ = """ """ // expected-error@-0{{invalid start of multi-line string literal}} // newline currently required after opening """ - -_ = "\(""" - not valid - """)" // expected-error@-2{{multi-line string literals are not allowed inside an interpolation}} -// expected-error@-3{{unterminated string literal}} -// expected-error@-2{{invalid start of multi-line string literal}} -// expected-error@-3{{unterminated string literal}} diff --git a/test/Parse/multiline_string.swift b/test/Parse/multiline_string.swift index 5705da75aa87a..fc0e36a3a3184 100644 --- a/test/Parse/multiline_string.swift +++ b/test/Parse/multiline_string.swift @@ -135,3 +135,28 @@ _ = """ _ = """ """ // CHECK: "" + +_ = "\(""" + \("a" + """ + valid + """) + """) literal" +// CHECK: "a" +// CHECK: " valid" +// CHECK: " literal" + +_ = "hello\(""" + world + """)" +// CHECK: "hello" +// CHECK: "world" + +_ = """ + hello\(""" + world + """) + abc + """ +// CHECK: "hello" +// CHECK: "world" +// CHECK: "\nabc" From b67d2ae3f0fb3e8bbf39c82a842e06dddef11ac0 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 00:41:30 +0100 Subject: [PATCH 2/6] multiline strings inside interpolations --- lib/Parse/Lexer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp index 794a66e8f9a0c..64b40d3ad3d57 100644 --- a/lib/Parse/Lexer.cpp +++ b/lib/Parse/Lexer.cpp @@ -1258,7 +1258,7 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, // issues with malformed tokens or other problems. switch (*CurPtr++) { // String literals in general cannot be split across multiple lines; - // interpolated ones are no exception (unless multiline strings.) + // interpolated ones are no exception (unless multiline strings). case '\n': case '\r': if (MultilineString) From aca8dfda45917723e2c33c7dc6e6a8056af72768 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 01:26:46 +0100 Subject: [PATCH 3/6] multiline strings inside interpolations --- lib/Parse/Lexer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp index 64b40d3ad3d57..eee9a43e99668 100644 --- a/lib/Parse/Lexer.cpp +++ b/lib/Parse/Lexer.cpp @@ -1258,7 +1258,7 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, // issues with malformed tokens or other problems. switch (*CurPtr++) { // String literals in general cannot be split across multiple lines; - // interpolated ones are no exception (unless multiline strings). + // interpolated ones are no exception (unless multiline literals). case '\n': case '\r': if (MultilineString) From 1125f69fa16d19f7841dbb54b95c56cf2c7a4618 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 19:21:55 +0100 Subject: [PATCH 4/6] buffer overrun protection --- lib/Parse/Lexer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp index eee9a43e99668..dec6225c0c440 100644 --- a/lib/Parse/Lexer.cpp +++ b/lib/Parse/Lexer.cpp @@ -1247,7 +1247,7 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, return !OpenDelimiters.empty() && (OpenDelimiters.back() == '"' || OpenDelimiters.back() == '\''); }; - while (true) { + while (CurPtr < EndPtr) { // This is a simple scanner, capable of recognizing nested parentheses and // string literals but not much else. The implications of this include not // being able to break an expression over multiple lines in an interpolated @@ -1258,7 +1258,7 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, // issues with malformed tokens or other problems. switch (*CurPtr++) { // String literals in general cannot be split across multiple lines; - // interpolated ones are no exception (unless multiline literals). + // interpolated ones are no exception - unless multiline literals. case '\n': case '\r': if (MultilineString) @@ -1332,6 +1332,9 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, continue; } } + + // Out of buffer. + return CurPtr-1; } /// getStringLiteralContent: From 17e9b62ab336aa1b175701cbd29808fff1c07b11 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 23:20:50 +0100 Subject: [PATCH 5/6] crash with SourceKit fixed --- a.swift | 166 ++++++++++++++++++++++++++++++++++++++++++++ lib/Parse/Lexer.cpp | 8 +-- 2 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 a.swift diff --git a/a.swift b/a.swift new file mode 100644 index 0000000000000..00d37b6d66aae --- /dev/null +++ b/a.swift @@ -0,0 +1,166 @@ +// +// AppDelegate.swift +// Multiline +// +// Created by John Holdsworth on 09/04/2017. +// Copyright © 2017 John Holdsworth. All rights reserved. +// + +import Cocoa + +@NSApplicationMain +class AppDelegate: NSObject, NSApplicationDelegate { + + + + func applicationDidFinishLaunching(_ aNotification: Notification) { + // Insert code here to initialize your application + +// print(""""hello world"""") + + var author = "john", xml = + "\n" + + "\n" + + " \n" + + " \(author)\n" + + " XML Developer's Guide\n" + + " Computer\n" + + " 44.95\n" + + " 2000-10-01\n" + + " An in-depth look at creating applications with XML.\n" + + " \n" + + "" + + print( """ + + + + \(author) + XML Developer's Guide + Computer + 44.95 + 2000-10-01 + An in-depth look at creating applications with XML. + + + """ ) + + assert( xml == """ + + + + \(author) + XML Developer's Guide + Computer + 44.95 + 2000-10-01 + An in-depth look at creating applications with XML. + + + """ ) + +// assert( xml == """ +// +// +// \(author) +// XML Developer's Guide +// Computer +// 44.95 +// 2000-10-01 +// An in-depth look at creating applications with XML. +// +// +// """ ) +// +// print( """ +// +// +// +// \(author) +// XML Developer's Guide +// Computer +// 44.95 +// 2000-10-01 +// An in-depth look at creating applications with XML. +// +// +// """ ) + +// let text = """ +// Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \ +// tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \ +// quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\ +// """ +// func delimit(_ str: String) -> String { +// return "<\(str)>" +// } +// print(delimit("""Nineteen\ +//Upsilon""")) + + print( """ + Usage: myapp + + Run myapp to do mything + + Options: + -myoption - an option + """ ) + +// let bookTuples = [("id", "john", "book", "novel", 9.99), +// ("id", "john", "book", "novel", 9.99)] +// +// xml = """ +// +// \n +// """ +// +// for (id, author, title, genre, price) in bookTuples { +// xml += """ +// +// \(author) +// \(title) +// \(genre) +// \(price) +// \n +// """ +// } +// +// xml += """ +// +// """ + + print( xml ) + + print("\((0..<10).map {""" + \("a" + """ + \($0) valid + """) + """}.joined(separator:" ")) literal") + + + let bookTuples = [(1, "john", "book", "novel", 9.99), + (2, "john", "book", "novel", 9.99)] + + print(""gsdf" + + + \(bookTuples.map { (id, author, title, genre, price) in """ + + \(author) + \(title) + \(genre) + \(price) + + """}.joined(separator:"\n")) + + """) + + } + + func applicationWillTerminate(_ aNotification: Notification) { + // Insert code here to tear down your application + } + + +} + diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp index dec6225c0c440..3146399c42391 100644 --- a/lib/Parse/Lexer.cpp +++ b/lib/Parse/Lexer.cpp @@ -1247,7 +1247,7 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, return !OpenDelimiters.empty() && (OpenDelimiters.back() == '"' || OpenDelimiters.back() == '\''); }; - while (CurPtr < EndPtr) { + while (true) { // This is a simple scanner, capable of recognizing nested parentheses and // string literals but not much else. The implications of this include not // being able to break an expression over multiple lines in an interpolated @@ -1332,9 +1332,6 @@ static const char *skipToEndOfInterpolatedExpression(const char *CurPtr, continue; } } - - // Out of buffer. - return CurPtr-1; } /// getStringLiteralContent: @@ -1492,7 +1489,8 @@ void Lexer::lexStringLiteral() { if (*CurPtr == '"' && *(CurPtr + 1) == '"' && *(CurPtr + 2) != '"') { CurPtr += 2; formToken(tok::string_literal, TokStart, MultilineString); - validateMultilineIndents(NextToken, Diags); + if (Diags) + validateMultilineIndents(NextToken, Diags); return; } else From 1a14e442191af31baf7fd7eb18c9bf8666ca5bf2 Mon Sep 17 00:00:00 2001 From: John Holdsworth Date: Thu, 27 Apr 2017 23:21:46 +0100 Subject: [PATCH 6/6] crash with SourceKit fixed --- a.swift | 166 -------------------------------------------------------- 1 file changed, 166 deletions(-) delete mode 100644 a.swift diff --git a/a.swift b/a.swift deleted file mode 100644 index 00d37b6d66aae..0000000000000 --- a/a.swift +++ /dev/null @@ -1,166 +0,0 @@ -// -// AppDelegate.swift -// Multiline -// -// Created by John Holdsworth on 09/04/2017. -// Copyright © 2017 John Holdsworth. All rights reserved. -// - -import Cocoa - -@NSApplicationMain -class AppDelegate: NSObject, NSApplicationDelegate { - - - - func applicationDidFinishLaunching(_ aNotification: Notification) { - // Insert code here to initialize your application - -// print(""""hello world"""") - - var author = "john", xml = - "\n" + - "\n" + - " \n" + - " \(author)\n" + - " XML Developer's Guide\n" + - " Computer\n" + - " 44.95\n" + - " 2000-10-01\n" + - " An in-depth look at creating applications with XML.\n" + - " \n" + - "" - - print( """ - - - - \(author) - XML Developer's Guide - Computer - 44.95 - 2000-10-01 - An in-depth look at creating applications with XML. - - - """ ) - - assert( xml == """ - - - - \(author) - XML Developer's Guide - Computer - 44.95 - 2000-10-01 - An in-depth look at creating applications with XML. - - - """ ) - -// assert( xml == """ -// -// -// \(author) -// XML Developer's Guide -// Computer -// 44.95 -// 2000-10-01 -// An in-depth look at creating applications with XML. -// -// -// """ ) -// -// print( """ -// -// -// -// \(author) -// XML Developer's Guide -// Computer -// 44.95 -// 2000-10-01 -// An in-depth look at creating applications with XML. -// -// -// """ ) - -// let text = """ -// Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \ -// tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \ -// quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\ -// """ -// func delimit(_ str: String) -> String { -// return "<\(str)>" -// } -// print(delimit("""Nineteen\ -//Upsilon""")) - - print( """ - Usage: myapp - - Run myapp to do mything - - Options: - -myoption - an option - """ ) - -// let bookTuples = [("id", "john", "book", "novel", 9.99), -// ("id", "john", "book", "novel", 9.99)] -// -// xml = """ -// -// \n -// """ -// -// for (id, author, title, genre, price) in bookTuples { -// xml += """ -// -// \(author) -// \(title) -// \(genre) -// \(price) -// \n -// """ -// } -// -// xml += """ -// -// """ - - print( xml ) - - print("\((0..<10).map {""" - \("a" + """ - \($0) valid - """) - """}.joined(separator:" ")) literal") - - - let bookTuples = [(1, "john", "book", "novel", 9.99), - (2, "john", "book", "novel", 9.99)] - - print(""gsdf" - - - \(bookTuples.map { (id, author, title, genre, price) in """ - - \(author) - \(title) - \(genre) - \(price) - - """}.joined(separator:"\n")) - - """) - - } - - func applicationWillTerminate(_ aNotification: Notification) { - // Insert code here to tear down your application - } - - -} -