diff --git a/.gitmodules b/.gitmodules index fd49688..0a4c3dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "libs/tracy"] path = libs/tracy url = git@github.com:wolfpld/tracy.git +[submodule "libs/fmt"] + path = libs/fmt + url = git@github.com:fmtlib/fmt.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c2f677..8ede9a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,11 @@ cmake_minimum_required(VERSION 3.22) project(typescript) -set(CMAKE_CXX_STANDARD 20) - -# enable for profiling -#add_definitions(-DTRACY_ENABLE) +set(CMAKE_CXX_STANDARD 23) include_directories(libs/tracy/) +include_directories(libs/fmt/include/) +include_directories(libs) list(APPEND CMAKE_MODULE_PATH libs/) @@ -14,8 +13,12 @@ list(APPEND CMAKE_MODULE_PATH libs/) #include_directories(libs/tracy/) add_subdirectory(libs/tracy) +add_subdirectory(libs/fmt) + +# enable for profiling +#add_definitions(-DTRACY_ENABLE) +#link_libraries(Tracy::TracyClient) -link_libraries(Tracy::TracyClient) add_subdirectory(src) include_directories(libs) diff --git a/libs/fmt b/libs/fmt new file mode 160000 index 0000000..9d60395 --- /dev/null +++ b/libs/fmt @@ -0,0 +1 @@ +Subproject commit 9d6039595302a6d6fe4cb2b8bfe98e9b09ed559b diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd4bd6b..d3c13b0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,16 +1,18 @@ cmake_minimum_required(VERSION 3.22) project(typescript) -add_definitions(-DTRACY_ENABLE) +#add_definitions(-DTRACY_ENABLE) set(CMAKE_CXX_FLAGS "-Wall -Wextra -O3 -ffast-math") +#set(CMAKE_CXX_FLAGS "-Wall -Wextra -O3 -ffast-math -fsanitize=leak") #set(CMAKE_CXX_FLAGS "-Wall -Wextra -O2") add_subdirectory(tests) - add_library(typescript utf.h utf.cpp core.h core.cpp utilities.h utilities.cpp node_test.h node_test.cpp syntax_cursor.h syntax_cursor.cpp parser2.h parser2.cpp types.h types.cpp path.h path.cpp factory.h factory.cpp parenthesizer.h parenthesizer.cpp scanner.h scanner.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../libs/tracy/TracyClient.cpp ) +# ${CMAKE_CURRENT_SOURCE_DIR}/../libs/tracy/TracyClient.cpp + +target_link_libraries(typescript fmt) \ No newline at end of file diff --git a/src/core.cpp b/src/core.cpp index 8a6c66e..4c5f8d9 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -96,25 +96,4 @@ namespace ts { while (s.size() > 0 && s.compare(s.size() - 1, 1, " ") == 0) s.erase(s.end() - 1); return s; } - std::string format(const std::string &fmt, ...) { - int size = ((int) fmt.size()) * 2 + 50; // Use a rubric appropriate for your code - std::string str; - va_list ap; - while (1) { // Maximum two passes on a POSIX system... - str.resize(size); - va_start(ap, fmt); - int n = vsnprintf((char *) str.data(), size, fmt.c_str(), ap); - va_end(ap); - if (n > -1 && n < size) { // Everything worked - str.resize(n); - return str; - } - if (n > -1) // Needed size returned - size = n + 1; // For null char - else - size *= 2; // Guess at a larger size (OS specific) - } - return str; - } - } \ No newline at end of file diff --git a/src/core.h b/src/core.h index 00d8506..d1a8b06 100644 --- a/src/core.h +++ b/src/core.h @@ -9,6 +9,7 @@ #include #include #include +#include #define CALLBACK(name) [this](auto ...a) { return name(a...); } @@ -36,7 +37,7 @@ namespace ts { return const_hash(input.c_str()); } - inline constexpr unsigned operator ""_hash(const char *s, size_t) { + inline consteval unsigned operator ""_hash(const char *s, size_t) { return const_hash(s); } @@ -242,10 +243,9 @@ namespace ts { return map.find(key) != map.end(); }; - std::string format(const std::string &fmt, ...); - - template - inline void debug(const std::string &fmt, Args &&...args) { - std::cout << format(fmt, args...) << "\n"; + template + inline void debug(T fmt, Args &&...args) { +// std::cout << fmt::format(fmt, std::forward(args)...) << "\n"; + std::cout << fmt::format(fmt, args...) << "\n"; } } diff --git a/src/diagnostic_messages.h b/src/diagnostic_messages.h index 978ee3c..e13f454 100644 --- a/src/diagnostic_messages.h +++ b/src/diagnostic_messages.h @@ -2,14 +2,15 @@ #include "types.h" #include +#include #include namespace ts { using types::DiagnosticMessage; using types::DiagnosticCategory; - static DiagnosticMessage diag(int code, DiagnosticCategory category, const std::string &key, const std::string &message, bool reportsUnnecessary = false, bool elidedInCompatabilityPyramid = false, bool reportsDeprecated = false) { - return DiagnosticMessage{code, category, key, message, reportsUnnecessary, elidedInCompatabilityPyramid, reportsDeprecated}; + inline shared diag(int code, DiagnosticCategory category, const std::string_view &key, const std::string_view &message, bool reportsUnnecessary = false, bool elidedInCompatabilityPyramid = false, bool reportsDeprecated = false) { + return std::make_shared(code, category, key, message, reportsUnnecessary, elidedInCompatabilityPyramid, reportsDeprecated); } class Diagnostics { diff --git a/src/factory.cpp b/src/factory.cpp index 2a3b051..c76b917 100644 --- a/src/factory.cpp +++ b/src/factory.cpp @@ -1,4 +1,5 @@ #include "factory.h" +#include namespace ts { int Factory::propagatePropertyNameFlagsOfChild(shared &node, int transformFlags) { @@ -18,11 +19,11 @@ namespace ts { return propagateChildFlags(std::move(node)) & ~(int) TransformFlags::ContainsPossibleTopLevelAwait; } - int Factory::propagateChildrenFlags(sharedOpt children) { + int Factory::propagateChildrenFlags(const sharedOpt &children) { return children ? children->transformFlags : (int) TransformFlags::None; } - void Factory::aggregateChildrenFlags(shared &children) { + void Factory::aggregateChildrenFlags(const shared &children) { int subtreeFlags = (int) TransformFlags::None; for (auto &&child: children->list) { subtreeFlags |= propagateChildFlags(child); @@ -30,30 +31,29 @@ namespace ts { children->transformFlags = subtreeFlags; } - shared Factory::createNodeArray(sharedOpt elements, optional hasTrailingComma) { + shared Factory::createNodeArray(const sharedOpt &elements, bool hasTrailingComma) { ZoneScoped; - if (elements) { - if (!hasTrailingComma || elements->hasTrailingComma == *hasTrailingComma) { - // Ensure the transform flags have been aggregated for this NodeArray - if (elements->transformFlags == (int) types::TransformFlags::None) { - aggregateChildrenFlags(elements); - } -// Debug.attachNodeArrayDebugInfo(elements); - return elements; + if (elements && elements->hasTrailingComma == hasTrailingComma) { + // Ensure the transform flags have been aggregated for this NodeArray + if (elements->transformFlags == (int) types::TransformFlags::None) { + aggregateChildrenFlags(elements); } +// Debug.attachNodeArrayDebugInfo(elements); + return elements; } - + elements->hasTrailingComma = hasTrailingComma; + return elements; // This *was* a `NodeArray`, but the `hasTrailingComma` option differs. Recreate the // array with the same elements, text range, and transform flags but with the updated // value for `hasTrailingComma` - auto array = make_shared(); - if (elements) array = elements; - array->pos = elements->pos; - array->end = elements->end; - array->hasTrailingComma = *hasTrailingComma; - array->transformFlags = elements->transformFlags; -// Debug.attachNodeArrayDebugInfo(array); - return array; +// auto array = make_shared(); +// if (elements) array = elements; +// array->pos = elements->pos; +// array->end = elements->end; +// array->hasTrailingComma = *hasTrailingComma; +// array->transformFlags = elements->transformFlags; +//// Debug.attachNodeArrayDebugInfo(array); +// return array; } sharedOpt Factory::asNodeArray(sharedOpt elements) { @@ -2498,7 +2498,7 @@ namespace ts { node->transformFlags |= (int) TransformFlags::ContainsESNext; break; default: - throw runtime_error(format("invalid keyword token %d", keywordToken)); + throw runtime_error(fmt::format("invalid keyword token {}", keywordToken)); } return node; } @@ -2555,7 +2555,7 @@ namespace ts { } // @api - shared Factory::createVariableDeclarationList(shared declarations, int flags) { + shared Factory::createVariableDeclarationList(const shared &declarations, int flags) { auto node = createBaseNode(SyntaxKind::VariableDeclarationList); node->flags |= flags & (int) NodeFlags::BlockScoped; node->declarations = declarations; @@ -2570,10 +2570,6 @@ namespace ts { return node; } - shared Factory::createVariableDeclarationList(vector> declarations, int flags) { - return createVariableDeclarationList(createNodeArray(declarations), flags); - } - // // @api // function updateBlock(node: Block, statements: readonly Statement[]) { // return node->statements != statements @@ -2582,14 +2578,9 @@ namespace ts { // } // @api - shared Factory::createVariableStatement(sharedOpt modifiers, variant, vector>> declarationList) { + shared Factory::createVariableStatement(sharedOpt modifiers, shared declarationList) { auto node = createBaseDeclaration(SyntaxKind::VariableStatement, /*decorators*/ nullptr, modifiers); - - if (holds_alternative>(declarationList)) { - node->declarationList = get>(declarationList); - } else { - node->declarationList = createVariableDeclarationList(get>>(declarationList)); - } + node->declarationList = declarationList; node->transformFlags |= propagateChildFlags(node->declarationList); if (modifiersToFlags(node->modifiers) & (int) ModifierFlags::Ambient) { @@ -2598,6 +2589,23 @@ namespace ts { return node; } +// // @api +// shared Factory::createVariableStatement(sharedOpt modifiers, variant, vector>> declarationList) { +// auto node = createBaseDeclaration(SyntaxKind::VariableStatement, /*decorators*/ nullptr, modifiers); +// +// if (holds_alternative>(declarationList)) { +// node->declarationList = get>(declarationList); +// } else { +// node->declarationList = createVariableDeclarationList(get>>(declarationList)); +// } +// node->transformFlags |= +// propagateChildFlags(node->declarationList); +// if (modifiersToFlags(node->modifiers) & (int) ModifierFlags::Ambient) { +// node->transformFlags = (int) TransformFlags::ContainsTypeScript; +// } +// return node; +// } + // // @api // function updateVariableStatement(node: VariableStatement, sharedOpt modifiers, declarationList: VariableDeclarationList) { // return node->modifiers != modifiers @@ -2933,6 +2941,7 @@ namespace ts { // // @api shared Factory::createVariableDeclaration(NameType name, sharedOpt exclamationToken, sharedOpt type, sharedOpt initializer) { + ZoneScoped; auto node = createBaseVariableLikeDeclaration( SyntaxKind::VariableDeclaration, /*decorators*/ nullptr, diff --git a/src/factory.h b/src/factory.h index 660d292..922464b 100644 --- a/src/factory.h +++ b/src/factory.h @@ -41,20 +41,20 @@ namespace ts { int propagateIdentifierNameFlags(shared node); - int propagateChildrenFlags(sharedOpt children); + int propagateChildrenFlags(const sharedOpt &children); - void aggregateChildrenFlags(shared &children); + void aggregateChildrenFlags(const shared &children); - shared createNodeArray(sharedOpt elements, optional hasTrailingComma = {}); + shared createNodeArray(const sharedOpt &elements, bool hasTrailingComma = false); sharedOpt asNodeArray(sharedOpt elements); - template - inline shared asNodeArray(const vector> &array) { - auto nodeArray = make_shared(); - for (auto &&node: array) nodeArray->list.push_back(node); - return nodeArray; - } +// template +// inline shared asNodeArray(const vector> &array) { +// auto nodeArray = make_shared(); +// for (auto &&node: array) nodeArray->list.push_back(node); +// return nodeArray; +// } // template // sharedOpt asNodeArray(optional>> &array) { @@ -65,7 +65,7 @@ namespace ts { // @api template - shared createNodeArray(const vector> &elements, optional hasTrailingComma = {}) { + shared createNodeArray(const vector> &elements, bool hasTrailingComma = false) { // Since the element list of a node array is typically created by starting with an empty array and // repeatedly calling push(), the list may not have the optimal memory layout. We invoke slice() for // small arrays (1 to 4 elements) to give the VM a chance to allocate an optimal representation. @@ -128,6 +128,7 @@ namespace ts { // function createToken(token: TKind): Token; template shared createToken(SyntaxKind token) { + ZoneScoped // Debug::asserts(token >= SyntaxKind::FirstToken && token <= SyntaxKind::LastToken, "Invalid token"); // Debug::asserts(token <= SyntaxKind::FirstTemplateToken || token >= SyntaxKind::LastTemplateToken, "Invalid token. Use 'createTemplateLiteralLikeNode' to create template literals."); // Debug::asserts(token <= SyntaxKind::FirstLiteralToken || token >= SyntaxKind::LastLiteralToken, "Invalid token. Use 'createLiteralLikeNode' to create literals."); @@ -1873,9 +1874,7 @@ namespace ts { shared createBlock(shared statements, bool multiLine); // @api - shared createVariableDeclarationList(shared declarations, int flags = (int) NodeFlags::None); - - shared createVariableDeclarationList(vector> declarations, int flags = (int) NodeFlags::None); + shared createVariableDeclarationList(const shared &declarations, int flags = (int) NodeFlags::None); // // @api // function updateBlock(node: Block, statements: readonly Statement[]) { @@ -1885,7 +1884,9 @@ namespace ts { // } // @api - shared createVariableStatement(sharedOpt modifiers, variant, vector>> declarationList); + shared createVariableStatement(sharedOpt modifiers, shared declarationList); + +// shared createVariableStatement(sharedOpt modifiers, variant, vector>> declarationList); // // @api // function updateVariableStatement(node: VariableStatement, sharedOpt modifiers, declarationList: VariableDeclarationList) { diff --git a/src/node_test.cpp b/src/node_test.cpp index 88453f4..d4b8bc3 100644 --- a/src/node_test.cpp +++ b/src/node_test.cpp @@ -1,4 +1,5 @@ #include "node_test.h" +#include namespace ts { /** @@ -127,7 +128,7 @@ namespace ts { return reinterpret_pointer_cast(node); } - throw runtime_error(format("resolveNamToNode with kind %d no valid name property", (int) node->kind)); + throw runtime_error(fmt::format("resolveNamToNode with kind {} no valid name property", (int) node->kind)); } bool isFunctionOrConstructorTypeNode(shared node) { @@ -168,7 +169,7 @@ namespace ts { case SyntaxKind::IndexSignature: return node->to().typeParameters; default: - throw runtime_error(format("node %d has no typeParameters", node->kind)); + throw runtime_error(fmt::format("node {} has no typeParameters", node->kind)); } } @@ -179,7 +180,7 @@ namespace ts { case SyntaxKind::JsxOpeningElement: return node->to().tagName; default: - throw runtime_error(format("node %d has no tagName", node->kind)); + throw runtime_error(fmt::format("node {} has no tagName", node->kind)); } } @@ -191,7 +192,7 @@ namespace ts { return reinterpret_pointer_cast(node)->escapedText; } - throw runtime_error(format("getEscapedName with kind %d no valid", (int) node->kind)); + throw runtime_error(fmt::format("getEscapedName with kind {} no valid", (int) node->kind)); } sharedOpt getName(const shared &node) { diff --git a/src/parser2.cpp b/src/parser2.cpp index c19132f..22f4e73 100644 --- a/src/parser2.cpp +++ b/src/parser2.cpp @@ -10,7 +10,7 @@ namespace ts { return currentToken = scanner.scan(); } - optional Parser::parseErrorAtPosition(int start, int length, const DiagnosticMessage &message, DiagnosticArg arg) { + optional Parser::parseErrorAtPosition(int start, int length, const shared &message, DiagnosticArg arg) { ZoneScoped; auto lastError = lastOrUndefined(parseDiagnostics); diff --git a/src/parser2.h b/src/parser2.h index ce0b8f8..097bd6d 100644 --- a/src/parser2.h +++ b/src/parser2.h @@ -16,6 +16,7 @@ #include "factory.h" #include "utilities.h" #include "diagnostic_messages.h" +#include using namespace ts::types; @@ -983,13 +984,13 @@ namespace ts { SyntaxKind nextTokenWithoutCheck(); - optional parseErrorAtPosition(int start, int length, const DiagnosticMessage &message, DiagnosticArg arg = ""); + optional parseErrorAtPosition(int start, int length, const shared &message, DiagnosticArg arg = ""); - optional parseErrorAt(int start, int end, const DiagnosticMessage &message, DiagnosticArg arg = "") { + optional parseErrorAt(int start, int end, const shared &message, DiagnosticArg arg = "") { return parseErrorAtPosition(start, end - start, message, arg); } - void scanError(const DiagnosticMessage &message, int length) { + void scanError(const shared &message, int length) { parseErrorAtPosition(scanner.getTextPos(), length, message); } @@ -1294,7 +1295,7 @@ namespace ts { setContextFlag(val, NodeFlags::AwaitContext); } - optional parseErrorAtCurrentToken(DiagnosticMessage message, DiagnosticArg arg = "") { + optional parseErrorAtCurrentToken(const shared &message, DiagnosticArg arg = "") { return parseErrorAt(scanner.getTokenPos(), scanner.getTextPos(), message, arg); } // @@ -1313,7 +1314,7 @@ namespace ts { // return result; // } - void parseErrorAtRange(shared range, DiagnosticMessage message, DiagnosticArg arg = "") { + void parseErrorAtRange(shared range, const shared &message, DiagnosticArg arg = "") { parseErrorAt(range->pos, range->end, message, arg); } @@ -1362,15 +1363,15 @@ namespace ts { ZoneScoped; // Keep track of the state we'll need to rollback to if lookahead fails (or if the // caller asked us to always reset our state). - auto saveToken = currentToken; - auto saveParseDiagnosticsLength = parseDiagnostics.size(); - auto saveParseErrorBeforeNextFinishedNode = parseErrorBeforeNextFinishedNode; + const auto saveToken = currentToken; + const auto saveParseDiagnosticsLength = parseDiagnostics.size(); + const auto saveParseErrorBeforeNextFinishedNode = parseErrorBeforeNextFinishedNode; // Note: it is not actually necessary to save/restore the context flags here. That's // because the saving/restoring of these flags happens naturally through the recursive // descent nature of our parser. However, we still store this here just so we can // assert that invariant holds. - auto saveContextFlags = contextFlags; + const auto saveContextFlags = contextFlags; // If we're only looking ahead, then tell the scanner to only lookahead as well. // Otherwise, if we're actually speculatively parsing, then tell the scanner to do the @@ -1383,7 +1384,7 @@ namespace ts { // If our callback returned something 'falsy' or we're just looking ahead, // then unconditionally restore us to where we were. - if (!(bool) result || (speculationKind != SpeculationKind::TryParse)) { + if ((speculationKind != SpeculationKind::TryParse) || !(bool) result) { currentToken = saveToken; if (speculationKind != SpeculationKind::Reparse) { parseDiagnostics.resize(saveParseDiagnosticsLength); @@ -1446,7 +1447,7 @@ namespace ts { * @param nameDiagnostic Diagnostic to report for all other cases. * @param tokenIfBlankName Current token if the name was invalid for being blank (not provided / skipped). */ - void parseErrorForInvalidName(DiagnosticMessage nameDiagnostic, DiagnosticMessage blankDiagnostic, SyntaxKind tokenIfBlankName) { + void parseErrorForInvalidName(shared nameDiagnostic, shared blankDiagnostic, SyntaxKind tokenIfBlankName) { if (token() == tokenIfBlankName) { parseErrorAtCurrentToken(blankDiagnostic); } else { @@ -1666,32 +1667,36 @@ namespace ts { // nextTokenJSDoc(); // return finishNode(factory.createToken(kind), pos) as T; // } -// + bool parseSemicolon() { return tryParseSemicolon() || parseExpected(SyntaxKind::SemicolonToken); } - shared createNodeArray(const vector> &elements, int pos, optional end = {}, optional hasTrailingComma = {}) { - auto array = factory.createNodeArray(elements, hasTrailingComma); - setTextRangePosEnd(array, pos, end ? *end : scanner.getStartPos()); - return array; - } - - //for empty call: createNodeArray(nullptr) - shared createNodeArray(vector> *elements, int pos, optional end = {}, optional hasTrailingComma = {}) { - auto array = factory.createNodeArray(elements ? *elements : (vector>) {}, hasTrailingComma); + shared createNodeArray(const shared &array, int pos, optional end = {}, bool hasTrailingComma = false) { setTextRangePosEnd(array, pos, end ? *end : scanner.getStartPos()); return array; } +// shared createNodeArray(const vector> &elements, int pos, optional end = {}, bool hasTrailingComma = false) { +// auto array = factory.createNodeArray(elements, hasTrailingComma); +// setTextRangePosEnd(array, pos, end ? *end : scanner.getStartPos()); +// return array; +// } +// +// //for empty call: createNodeArray(nullptr) +// shared createNodeArray(vector> *elements, int pos, optional end = {}, bool hasTrailingComma = false) { +// auto array = factory.createNodeArray(elements ? *elements : (vector>) {}, hasTrailingComma); +// setTextRangePosEnd(array, pos, end ? *end : scanner.getStartPos()); +// return array; +// } // function createMissingNode(kind: T["kind"], reportAtCurrentPosition: false, optional diagnosticMessage, arg0?: any): T; // function createMissingNode(kind: T["kind"], reportAtCurrentPosition: boolean, diagnosticMessage: DiagnosticMessage, arg0?: any): T; template - shared createMissingNode(SyntaxKind kind, bool reportAtCurrentPosition, optional diagnosticMessage = {}, DiagnosticArg arg = "") { + shared createMissingNode(SyntaxKind kind, bool reportAtCurrentPosition, const sharedOpt &diagnosticMessage = nullptr, DiagnosticArg arg = "") { if (reportAtCurrentPosition && diagnosticMessage) { - parseErrorAtPosition(scanner.getStartPos(), 0, *diagnosticMessage, arg); + parseErrorAtPosition(scanner.getStartPos(), 0, diagnosticMessage, arg); } else if (diagnosticMessage) { - parseErrorAtCurrentToken(*diagnosticMessage, arg); + parseErrorAtCurrentToken(diagnosticMessage, arg); } auto pos = getNodePos(); @@ -1705,8 +1710,10 @@ namespace ts { return to(finishNode(result, pos)); } + string internIdentifier(const string &text) { - //todo: add back + //this was used in the JS version as optimization to not reuse text instances. + //we do not need that. return text; // auto identifier = get(identifiers, text); // if (!identifier) { @@ -1726,12 +1733,12 @@ namespace ts { return token() > SyntaxKind::LastReservedWord; } - shared parseBindingIdentifier(optional privateIdentifierDiagnosticMessage = {}) { + shared parseBindingIdentifier(const sharedOpt &privateIdentifierDiagnosticMessage = nullptr) { ZoneScoped; return createIdentifier(isBindingIdentifier(), /*diagnosticMessage*/ {}, privateIdentifierDiagnosticMessage); } - shared parseIdentifierName(optional diagnosticMessage = {}) { + shared parseIdentifierName(const sharedOpt &diagnosticMessage = nullptr) { return createIdentifier(tokenIsIdentifierOrKeyword(token()), diagnosticMessage); } @@ -2812,12 +2819,12 @@ namespace ts { // return parameter.initializer == undefined; // } // - optional getExpectedCommaDiagnostic(ParsingContext kind) { + sharedOpt getExpectedCommaDiagnostic(ParsingContext kind) { if (kind == ParsingContext::EnumMembers) return Diagnostics::An_enum_member_name_must_be_followed_by_a_or; - return nullopt; + return nullptr; } - bool parseExpected(SyntaxKind kind, optional diagnosticMessage = {}, bool shouldAdvance = true) { + bool parseExpected(SyntaxKind kind, const sharedOpt &diagnosticMessage = nullptr, bool shouldAdvance = true) { if (token() == kind) { if (shouldAdvance) { nextToken(); @@ -2827,7 +2834,7 @@ namespace ts { // Report specific message if provided with one. Otherwise, report generic fallback message. if (diagnosticMessage) { - parseErrorAtCurrentToken(*diagnosticMessage); + parseErrorAtCurrentToken(diagnosticMessage); } else { parseErrorAtCurrentToken(Diagnostics::_0_expected, tokenToString(kind)); } @@ -2934,15 +2941,14 @@ namespace ts { ZoneScoped; int saveParsingContext = parsingContext; parsingContext |= 1 << (int) kind; - vector> list; - + auto list = make_shared(); auto listPos = getNodePos(); while (!isListTerminator(kind)) { if (isListElement(kind, /*inErrorRecovery*/ false)) { auto n = parseListElement(kind, parseElement); if (!n) throw runtime_error("No node given"); - list.push_back(n); + list->push(n); continue; } @@ -2963,7 +2969,7 @@ namespace ts { ZoneScoped; auto saveParsingContext = parsingContext; parsingContext |= 1 << (int) kind; - vector> list; + auto list = make_shared(); auto listPos = getNodePos(); int commaStart = -1; // Meaning the previous token was not a comma @@ -2975,7 +2981,7 @@ namespace ts { parsingContext = saveParsingContext; return nullptr; } - list.push_back(result); + list->push(result); commaStart = scanner.getTokenPos(); if (parseOptional(SyntaxKind::CommaToken)) { @@ -3046,7 +3052,7 @@ namespace ts { // An identifier that starts with two underscores has an extra underscore character prepended to it to avoid issues // with magic property names like '__proto__'. The 'identifiers' object is used to share a single string instance for // each identifier in order to reduce memory consumption. - shared createIdentifier(bool isIdentifier, optional diagnosticMessage = {}, optional privateIdentifierDiagnosticMessage = {}) { + shared createIdentifier(bool isIdentifier, const sharedOpt &diagnosticMessage = nullptr, const sharedOpt &privateIdentifierDiagnosticMessage = nullptr) { ZoneScoped; if (isIdentifier) { identifierCount++; @@ -3059,7 +3065,7 @@ namespace ts { } if (token() == SyntaxKind::PrivateIdentifier) { - parseErrorAtCurrentToken(privateIdentifierDiagnosticMessage ? *privateIdentifierDiagnosticMessage : Diagnostics::Private_identifiers_are_not_allowed_outside_class_bodies); + parseErrorAtCurrentToken(privateIdentifierDiagnosticMessage ? privateIdentifierDiagnosticMessage : Diagnostics::Private_identifiers_are_not_allowed_outside_class_bodies); return createIdentifier(/*isIdentifier*/ true); } @@ -3079,10 +3085,10 @@ namespace ts { Diagnostics::Identifier_expected_0_is_a_reserved_word_that_cannot_be_used_here : Diagnostics::Identifier_expected; - return createMissingNode(SyntaxKind::Identifier, reportAtCurrentPosition, diagnosticMessage ? *diagnosticMessage : defaultMessage); + return createMissingNode(SyntaxKind::Identifier, reportAtCurrentPosition, diagnosticMessage ? diagnosticMessage : defaultMessage); } - shared parseIdentifier(optional diagnosticMessage = {}, optional privateIdentifierDiagnosticMessage = {}) { + shared parseIdentifier(const sharedOpt &diagnosticMessage = nullptr, const sharedOpt &privateIdentifierDiagnosticMessage = nullptr) { return createIdentifier(isIdentifier(), diagnosticMessage, privateIdentifierDiagnosticMessage); } @@ -3199,9 +3205,9 @@ namespace ts { // function parseExpectedToken(t: TKind, optional diagnosticMessage, arg0?: any): Token; template - shared parseExpectedToken(SyntaxKind t, optional diagnosticMessage = {}, DiagnosticArg arg0 = "") { + shared parseExpectedToken(SyntaxKind t, const sharedOpt &diagnosticMessage = nullptr, DiagnosticArg arg0 = "") { if (auto a = parseOptionalToken(t)) return a; - return createMissingNode(t, /*reportAtCurrentPosition*/ false, diagnosticMessage ? *diagnosticMessage : Diagnostics::_0_expected, arg0 != "" ? arg0 : tokenToString(t)); + return createMissingNode(t, /*reportAtCurrentPosition*/ false, diagnosticMessage ? diagnosticMessage : Diagnostics::_0_expected, arg0 != "" ? arg0 : tokenToString(t)); } shared parseLiteralOfTemplateSpan(bool isTaggedTemplate) { @@ -3227,11 +3233,11 @@ namespace ts { shared parseTemplateSpans(bool isTaggedTemplate) { auto pos = getNodePos(); - vector> list; + auto list = make_shared(); sharedOpt node; do { node = parseTemplateSpan(isTaggedTemplate); - list.push_back(node); + list->push(node); } while (node->literal->kind == SyntaxKind::TemplateMiddle); return createNodeArray(list, pos); } @@ -3256,7 +3262,7 @@ namespace ts { ); } - shared parseEntityName(bool allowReservedWords, optional diagnosticMessage = {}) { + shared parseEntityName(bool allowReservedWords, const sharedOpt &diagnosticMessage = nullptr) { auto pos = getNodePos(); shared entity = allowReservedWords ? parseIdentifierName(diagnosticMessage) : parseIdentifier(diagnosticMessage); auto dotPos = getNodePos(); @@ -3639,12 +3645,12 @@ namespace ts { ZoneScoped; auto pos = getNodePos(); sharedOpt decorator; - vector> list; -// auto list, decorator; + sharedOpt list; while (decorator = tryParseDecorator()) { - list = append(list, decorator); + if (!list) list = make_shared(); + list->push(decorator); } - if (list.empty()) return nullptr; + if (!list) return nullptr; return createNodeArray(list, pos); } @@ -3654,8 +3660,8 @@ namespace ts { sharedOpt tryParseModifier(bool permitInvalidConstAsModifier = false, bool stopOnStartOfClassStaticBlock = false, bool hasSeenStaticModifier = false) { ZoneScoped; - auto pos = getNodePos(); - auto kind = token(); + const auto pos = getNodePos(); + const auto kind = token(); if (token() == SyntaxKind::ConstKeyword && permitInvalidConstAsModifier) { // We need to ensure that any subsequent modifiers appear on the same line @@ -3685,15 +3691,16 @@ namespace ts { */ sharedOpt parseModifiers(bool permitInvalidConstAsModifier = false, bool stopOnStartOfClassStaticBlock = false) { ZoneScoped; - auto pos = getNodePos(); + const auto pos = getNodePos(); auto hasSeenStatic = false; - vector> list; + sharedOpt list; sharedOpt modifier; while ((modifier = tryParseModifier(permitInvalidConstAsModifier, stopOnStartOfClassStaticBlock, hasSeenStatic))) { if (modifier->kind == SyntaxKind::StaticKeyword) hasSeenStatic = true; - list = append(list, modifier); + if (!list) list = make_shared(); + list->push(modifier); } - if (!list.empty()) return createNodeArray(list, pos); + if (list) return createNodeArray(list, pos); return nullptr; } @@ -3963,7 +3970,7 @@ namespace ts { return finishNode(factory.createObjectBindingPattern(elements), pos); } - shared> parseIdentifierOrPattern(optional privateIdentifierDiagnosticMessage = {}) { + shared> parseIdentifierOrPattern(const shared &privateIdentifierDiagnosticMessage = nullptr) { ZoneScoped; if (token() == SyntaxKind::OpenBracketToken) { return parseArrayBindingPattern(); @@ -4042,7 +4049,9 @@ namespace ts { auto pos = getNodePos(); nextToken(); auto modifier = finishNode(factory.createToken(SyntaxKind::AbstractKeyword), pos); - modifiers = createNodeArray({modifier}, pos); + auto list = make_shared(); + list->push(modifier); + modifiers = createNodeArray(list, pos); } return modifiers; } @@ -4214,7 +4223,7 @@ namespace ts { case SyntaxKind::LessThanToken: return parseJsxElementOrSelfClosingElementOrFragment(/*inExpressionContext*/ false, /*topInvalidNodePosition*/ {}, openingTag); default: - throw runtime_error(format("Should not reach %d", token)); + throw runtime_error(fmt::format("Should not reach {}", token)); // return Debug::assertsNever(token); } } @@ -4246,7 +4255,7 @@ namespace ts { } shared parseJsxChildren(shared openingTag) { - vector> list; + auto list = make_shared(); auto listPos = getNodePos(); auto saveParsingContext = parsingContext; parsingContext |= 1 << (int) ParsingContext::JsxChildren; @@ -4254,7 +4263,7 @@ namespace ts { while (true) { auto child = parseJsxChild(openingTag, currentToken = scanner.reScanJsxToken()); if (!child) break; - list.push_back(child); + list->push(child); if (isJsxOpeningElement(openingTag) && (child && child->kind == SyntaxKind::JsxElement) && !tagNamesAreEquivalent(getTagName(child->to().openingElement), getTagName(child->to().closingElement)) @@ -4328,7 +4337,7 @@ namespace ts { auto c = children->slice(0, children->length() - 1); c.push_back(newLast); - children = createNodeArray(c, children->pos, end); + children = createNodeArray(make_shared(c), children->pos, end); closingElement = jsxElement.closingElement; } else { closingElement = parseJsxClosingElement(opening, inExpressionContext); @@ -4561,7 +4570,7 @@ namespace ts { } } - shared parseBlock(bool ignoreMissingOpenBrace, optional diagnosticMessage = {}) { + shared parseBlock(bool ignoreMissingOpenBrace, const sharedOpt &diagnosticMessage = nullptr) { auto pos = getNodePos(); auto hasJSDoc = hasPrecedingJSDocComment(); auto openBracePosition = scanner.getTokenPos(); @@ -4583,7 +4592,7 @@ namespace ts { } } - shared parseFunctionBlock(int flags, optional diagnosticMessage = {}) { + shared parseFunctionBlock(int flags, const sharedOpt &diagnosticMessage = nullptr) { auto savedYieldContext = inYieldContext(); setYieldContext(!!(flags & (int) SignatureFlags::Yield)); @@ -4613,7 +4622,7 @@ namespace ts { return block; } - sharedOpt parseFunctionBlockOrSemicolon(int flags, optional diagnosticMessage = {}) { + sharedOpt parseFunctionBlockOrSemicolon(int flags, const sharedOpt &diagnosticMessage = nullptr) { if (token() != SyntaxKind::OpenBraceToken && canParseSemicolon()) { parseSemicolon(); return nullptr; @@ -4859,7 +4868,7 @@ namespace ts { shared name, sharedOpt questionToken, sharedOpt exclamationToken, - optional diagnosticMessage = {} + const sharedOpt &diagnosticMessage = nullptr ) { auto isGenerator = asteriskToken ? SignatureFlags::Yield : SignatureFlags::None; auto isAsync = some(modifiers, isAsyncModifier) ? SignatureFlags::Await : SignatureFlags::None; @@ -5811,7 +5820,7 @@ namespace ts { auto elements = parseDelimitedList(ParsingContext::AssertEntries, CALLBACK(parseAssertEntry), /*considerSemicolonAsDelimiter*/ true); if (!parseExpected(SyntaxKind::CloseBraceToken)) { auto lastError = lastOrUndefined(parseDiagnostics); - if (lastError && lastError->code == Diagnostics::_0_expected.code) { + if (lastError && lastError->code == Diagnostics::_0_expected->code) { addRelatedInfo( *lastError, {createDetachedDiagnostic(fileName, openBracePosition, 1, Diagnostics::The_parser_expected_to_find_a_1_to_match_the_0_token_here, {"{", "}"})} @@ -5835,7 +5844,7 @@ namespace ts { auto clause = parseAssertClause(/*skipAssertKeyword*/ true); if (!parseExpected(SyntaxKind::CloseBraceToken)) { auto lastError = lastOrUndefined(parseDiagnostics); - if (lastError && lastError->code == Diagnostics::_0_expected.code) { + if (lastError && lastError->code == Diagnostics::_0_expected->code) { addRelatedInfo( *lastError, {createDetachedDiagnostic(fileName, openBracePosition, 1, Diagnostics::The_parser_expected_to_find_a_1_to_match_the_0_token_here, {"{", "}"})} @@ -5990,11 +5999,11 @@ namespace ts { shared parseTemplateTypeSpans() { auto pos = getNodePos(); - vector> list; + auto list = make_shared(); shared node; do { node = parseTemplateTypeSpan(); - list.push_back(node); + list->push(node); } while (node->literal->kind == SyntaxKind::TemplateMiddle); return createNodeArray(list, pos); } @@ -6144,7 +6153,7 @@ namespace ts { // try to parse them gracefully and issue a helpful message. if (isStartOfFunctionTypeOrConstructorType()) { auto type = parseFunctionOrConstructorType(); - DiagnosticMessage diagnostic; + shared diagnostic; if (isFunctionTypeNode(type)) { diagnostic = isInUnionType ? Diagnostics::Function_type_notation_must_be_parenthesized_when_used_in_a_union_type @@ -6170,15 +6179,15 @@ namespace ts { auto hasLeadingOperator = parseOptional(operatorKind); sharedOpt type = hasLeadingOperator ? parseFunctionOrConstructorTypeToError(isUnionType) : parseConstituentType(); if (token() == operatorKind || hasLeadingOperator) { - vector> types = {type}; + auto types = make_shared(type); while (parseOptional(operatorKind)) { if (auto a = parseFunctionOrConstructorTypeToError(isUnionType)) { - types.push_back(a); + types->push(a); } else { - types.push_back(parseConstituentType()); + types->push(parseConstituentType()); } } - type = finishNode(createTypeNode(createNodeArray(types, pos)), pos); + type = finishNode(createTypeNode(types), pos); } return type; } @@ -6192,6 +6201,7 @@ namespace ts { } shared parseType() { + ZoneScoped; if (contextFlags & (int) NodeFlags::TypeExcludesFlags) { return doOutsideOfContext>(NodeFlags::TypeExcludesFlags, CALLBACK(parseType)); } @@ -6214,6 +6224,7 @@ namespace ts { } sharedOpt parseTypeAnnotation() { + ZoneScoped; return parseOptional(SyntaxKind::ColonToken) ? parseType() : nullptr; } @@ -6528,7 +6539,9 @@ namespace ts { auto pos = getNodePos(); nextToken(); auto modifier = finishNode(factory.createToken(SyntaxKind::AsyncKeyword), pos); - return factory.createNodeArray({modifier}, pos); + auto list = make_shared(); + list->push(modifier); + return factory.createNodeArray(list, pos); } return nullptr; } @@ -6709,7 +6722,7 @@ namespace ts { ); finishNode(parameter, identifier->pos); - auto parameters = createNodeArray({parameter}, parameter->pos, parameter->end); + auto parameters = createNodeArray(make_shared(parameter), parameter->pos, parameter->end); auto equalsGreaterThanToken = parseExpectedToken(SyntaxKind::EqualsGreaterThanToken); auto body = parseArrowFunctionExpressionBody(/*isAsync*/ !!asyncModifier); @@ -6797,8 +6810,8 @@ namespace ts { // Otherwise, we try to parse out the conditional expression bit. We want to allow any // binary expression here, so we pass in the 'lowest' precedence here so that it matches // and consumes anything. - auto pos = getNodePos(); - auto expr = parseBinaryExpressionOrHigher((int) OperatorPrecedence::Lowest); + const auto pos = getNodePos(); + const auto expr = parseBinaryExpressionOrHigher((int) OperatorPrecedence::Lowest); // To avoid a look-ahead, we did not handle the case of an arrow function with a single un-parenthesized // parameter ('x => ...') above. We handle it here by checking if the parsed expression was a single @@ -7095,17 +7108,17 @@ namespace ts { shared parseVariableDeclaration(optional allowExclamation = {}) { ZoneScoped; - auto pos = getNodePos(); - auto hasJSDoc = hasPrecedingJSDocComment(); - auto name = parseIdentifierOrPattern(Diagnostics::Private_identifiers_are_not_allowed_in_variable_declarations); + const auto pos = getNodePos(); + const auto hasJSDoc = hasPrecedingJSDocComment(); + const auto name = parseIdentifierOrPattern(Diagnostics::Private_identifiers_are_not_allowed_in_variable_declarations); sharedOpt exclamationToken; if (isTrue(allowExclamation) && name->kind == SyntaxKind::Identifier && token() == SyntaxKind::ExclamationToken && !scanner.hasPrecedingLineBreak()) { exclamationToken = parseTokenNode(); } - auto type = parseTypeAnnotation(); + const auto type = parseTypeAnnotation(); sharedOpt initializer = isInOrOfKeyword(token()) ? nullptr : parseInitializer(); - auto node = factory.createVariableDeclaration(name, exclamationToken, type, initializer); + const auto node = factory.createVariableDeclaration(name, exclamationToken, type, initializer); return withJSDoc(finishNode(node, pos), hasJSDoc); } @@ -7204,7 +7217,6 @@ namespace ts { shared parseDeclaration() { ZoneScoped; - // TODO: Can we hold onto the parsed decorators/modifiers and advance the scanner // if we can't reuse the declaration, so that we don't do this work twice? // diff --git a/src/scanner.cpp b/src/scanner.cpp index bcc1099..702892f 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -37,6 +37,7 @@ namespace ts { } string Scanner::scanString(bool jsxAttributeString) { + ZoneScoped; auto quote = charCodeAt(text, pos); pos++; string result; @@ -85,6 +86,7 @@ namespace ts { } string Scanner::scanHexDigits(int minCount, bool scanAsManyAsPossible, bool canHaveSeparators) { + ZoneScoped; auto allowSeparator = false; auto isPreviousTokenSeparator = false; auto found = 0; @@ -146,6 +148,7 @@ namespace ts { } string Scanner::scanExtendedUnicodeEscape() { + ZoneScoped; auto escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); auto escapedValue = !escapedValueString.empty() ? stoi(escapedValueString, nullptr, 16) : -1; auto isInvalidExtendedEscape = false; @@ -179,6 +182,7 @@ namespace ts { } string Scanner::scanEscapeSequence(bool isTaggedTemplate) { + ZoneScoped; auto start = pos; pos++; if (pos >= end) { @@ -292,6 +296,7 @@ namespace ts { } SyntaxKind Scanner::scanTemplateAndSetTokenValue(bool isTaggedTemplate) { + ZoneScoped; auto startedWithBacktick = charCodeAt(text, pos).code == CharacterCodes::backtick; pos++; @@ -359,6 +364,7 @@ namespace ts { } string Scanner::scanNumberFragment() { + ZoneScoped; auto start = pos; auto allowSeparator = false; auto isPreviousTokenSeparator = false; @@ -394,89 +400,6 @@ namespace ts { return result + substring(text, start, pos); } - static map textToKeyword{ - {"abstract", SyntaxKind::AbstractKeyword}, - {"any", SyntaxKind::AnyKeyword}, - {"as", SyntaxKind::AsKeyword}, - {"asserts", SyntaxKind::AssertsKeyword}, - {"assert", SyntaxKind::AssertKeyword}, - {"bigint", SyntaxKind::BigIntKeyword}, - {"boolean", SyntaxKind::BooleanKeyword}, - {"break", SyntaxKind::BreakKeyword}, - {"case", SyntaxKind::CaseKeyword}, - {"catch", SyntaxKind::CatchKeyword}, - {"class", SyntaxKind::ClassKeyword}, - {"continue", SyntaxKind::ContinueKeyword}, - {"const", SyntaxKind::ConstKeyword}, - {"constructor", SyntaxKind::ConstructorKeyword}, - {"debugger", SyntaxKind::DebuggerKeyword}, - {"declare", SyntaxKind::DeclareKeyword}, - {"default", SyntaxKind::DefaultKeyword}, - {"delete", SyntaxKind::DeleteKeyword}, - {"do", SyntaxKind::DoKeyword}, - {"else", SyntaxKind::ElseKeyword}, - {"enum", SyntaxKind::EnumKeyword}, - {"export", SyntaxKind::ExportKeyword}, - {"extends", SyntaxKind::ExtendsKeyword}, - {"false", SyntaxKind::FalseKeyword}, - {"finally", SyntaxKind::FinallyKeyword}, - {"for", SyntaxKind::ForKeyword}, - {"from", SyntaxKind::FromKeyword}, - {"function", SyntaxKind::FunctionKeyword}, - {"get", SyntaxKind::GetKeyword}, - {"if", SyntaxKind::IfKeyword}, - {"implements", SyntaxKind::ImplementsKeyword}, - {"import", SyntaxKind::ImportKeyword}, - {"in", SyntaxKind::InKeyword}, - {"infer", SyntaxKind::InferKeyword}, - {"instanceof", SyntaxKind::InstanceOfKeyword}, - {"interface", SyntaxKind::InterfaceKeyword}, - {"intrinsic", SyntaxKind::IntrinsicKeyword}, - {"is", SyntaxKind::IsKeyword}, - {"keyof", SyntaxKind::KeyOfKeyword}, - {"let", SyntaxKind::LetKeyword}, - {"module", SyntaxKind::ModuleKeyword}, - {"namespace", SyntaxKind::NamespaceKeyword}, - {"never", SyntaxKind::NeverKeyword}, - {"new", SyntaxKind::NewKeyword}, - {"null", SyntaxKind::NullKeyword}, - {"number", SyntaxKind::NumberKeyword}, - {"object", SyntaxKind::ObjectKeyword}, - {"package", SyntaxKind::PackageKeyword}, - {"private", SyntaxKind::PrivateKeyword}, - {"protected", SyntaxKind::ProtectedKeyword}, - {"public", SyntaxKind::PublicKeyword}, - {"override", SyntaxKind::OverrideKeyword}, - {"out", SyntaxKind::OutKeyword}, - {"readonly", SyntaxKind::ReadonlyKeyword}, - {"require", SyntaxKind::RequireKeyword}, - {"global", SyntaxKind::GlobalKeyword}, - {"return", SyntaxKind::ReturnKeyword}, - {"set", SyntaxKind::SetKeyword}, - {"static", SyntaxKind::StaticKeyword}, - {"string", SyntaxKind::StringKeyword}, - {"super", SyntaxKind::SuperKeyword}, - {"switch", SyntaxKind::SwitchKeyword}, - {"symbol", SyntaxKind::SymbolKeyword}, - {"this", SyntaxKind::ThisKeyword}, - {"throw", SyntaxKind::ThrowKeyword}, - {"true", SyntaxKind::TrueKeyword}, - {"try", SyntaxKind::TryKeyword}, - {"type", SyntaxKind::TypeKeyword}, - {"typeof", SyntaxKind::TypeOfKeyword}, - {"undefined", SyntaxKind::UndefinedKeyword}, - {"unique", SyntaxKind::UniqueKeyword}, - {"unknown", SyntaxKind::UnknownKeyword}, - {"var", SyntaxKind::VarKeyword}, - {"void", SyntaxKind::VoidKeyword}, - {"while", SyntaxKind::WhileKeyword}, - {"with", SyntaxKind::WithKeyword}, - {"yield", SyntaxKind::YieldKeyword}, - {"async", SyntaxKind::AsyncKeyword}, - {"await", SyntaxKind::AwaitKeyword}, - {"of", SyntaxKind::OfKeyword}, - }; - /** * Test for whether a single line comment with leading whitespace trimmed's text contains a directive. */ @@ -492,6 +415,7 @@ namespace ts { const unsigned long mergeConflictMarkerLength = size("<<<<<<<") - 1; bool isConflictMarkerTrivia(const string &text, int pos) { + ZoneScoped; assert(pos >= 0); // Conflict markers must be at the start of a line. @@ -512,10 +436,10 @@ namespace ts { return false; } - int Scanner::error(const DiagnosticMessage &message, int errPos, int length) { + int Scanner::error(const shared &message, int errPos, int length) { if (errPos == -1) errPos = pos; - cout << "Error: " << message.code << ": " << message.message << " at " << errPos << "\n"; + cout << "Error: " << message->code << ": " << message->message << " at " << errPos << "\n"; if (onError) { (*onError)(message, length); @@ -523,6 +447,7 @@ namespace ts { } int scanConflictMarkerTrivia(string &text, int pos) { + ZoneScoped; auto ch = charCodeAt(text, pos); auto len = text.size(); @@ -553,6 +478,7 @@ namespace ts { } bool lookupInUnicodeMap(CharCode code, vector &map) { + ZoneScoped; // Bail out quickly if it couldn't possibly be in the map. if (code.code < map[0]) { return false; @@ -639,7 +565,7 @@ namespace ts { return isWhiteSpaceSingleLine(ch) || isLineBreak(ch); } -/* @internal */ + /* @internal */ int ts::skipTrivia(string &text, int pos, optional stopAfterLineBreak, optional stopAtComments, optional inJSDoc) { ZoneScoped; if (positionIsSynthesized(pos)) { @@ -814,6 +740,7 @@ namespace ts { } SyntaxKind Scanner::checkBigIntSuffix() { + ZoneScoped; if (charCodeAt(text, pos).code == CharacterCodes::n) { tokenValue += "n"; // Use base 10 instead of base 2 or base 8 for shorter literals @@ -830,7 +757,7 @@ namespace ts { : tokenFlags & TokenFlags::OctalSpecifier ? stoi(substring(tokenValue, 2), 0, 8) // skip "0o" : stoi(tokenValue); - tokenValue = "" + std::to_string(numericValue); + tokenValue = std::to_string(numericValue); return SyntaxKind::NumericLiteral; } } @@ -844,6 +771,7 @@ namespace ts { } SyntaxKind Scanner::scanJsxAttributeValue() { + ZoneScoped; startPos = pos; switch (charCodeAt(text, pos).code) { @@ -858,6 +786,7 @@ namespace ts { } SyntaxKind Scanner::scanJsxIdentifier() { + ZoneScoped; if (tokenIsIdentifierOrKeyword(token)) { // An identifier or keyword has already been parsed - check for a `-` or a single instance of `:` and then append it and // everything after it to the token @@ -894,6 +823,7 @@ namespace ts { } SyntaxKind Scanner::scanJsxToken(bool allowMultilineJsxText) { + ZoneScoped; startPos = tokenPos = pos; if (pos >= end) { @@ -998,6 +928,7 @@ namespace ts { } ScanNumber Scanner::scanNumber() { + ZoneScoped; auto start = pos; auto mainFragment = scanNumberFragment(); string decimalFragment; @@ -1052,6 +983,7 @@ namespace ts { } string Scanner::scanBinaryOrOctalDigits(int base) { + ZoneScoped; string value; // For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b. // Similarly valid octalIntegerLiteral must have at least one octal digit following o or O. @@ -1089,21 +1021,186 @@ namespace ts { } SyntaxKind Scanner::getIdentifierToken() { + ZoneScoped; // Reserved words are between 2 and 12 characters long and start with a lowercase letter auto len = tokenValue.size(); if (len >= 2 && len <= 12) { auto ch = charCodeAt(tokenValue, 0); if (ch.code >= CharacterCodes::a && ch.code <= CharacterCodes::z) { - auto it = textToKeyword.find(tokenValue); - if (it != textToKeyword.end()) { - return token = it->second; + switch (const_hash(tokenValue)) { + case "abstract"_hash: + return SyntaxKind::AbstractKeyword; + case "any"_hash: + return SyntaxKind::AnyKeyword; + case "as"_hash: + return SyntaxKind::AsKeyword; + case "asserts"_hash: + return SyntaxKind::AssertsKeyword; + case "assert"_hash: + return SyntaxKind::AssertKeyword; + case "bigint"_hash: + return SyntaxKind::BigIntKeyword; + case "boolean"_hash: + return SyntaxKind::BooleanKeyword; + case "break"_hash: + return SyntaxKind::BreakKeyword; + case "case"_hash: + return SyntaxKind::CaseKeyword; + case "catch"_hash: + return SyntaxKind::CatchKeyword; + case "class"_hash: + return SyntaxKind::ClassKeyword; + case "continue"_hash: + return SyntaxKind::ContinueKeyword; + case "const"_hash: + return SyntaxKind::ConstKeyword; + case "constructor"_hash: + return SyntaxKind::ConstructorKeyword; + case "debugger"_hash: + return SyntaxKind::DebuggerKeyword; + case "declare"_hash: + return SyntaxKind::DeclareKeyword; + case "default"_hash: + return SyntaxKind::DefaultKeyword; + case "delete"_hash: + return SyntaxKind::DeleteKeyword; + case "do"_hash: + return SyntaxKind::DoKeyword; + case "else"_hash: + return SyntaxKind::ElseKeyword; + case "enum"_hash: + return SyntaxKind::EnumKeyword; + case "export"_hash: + return SyntaxKind::ExportKeyword; + case "extends"_hash: + return SyntaxKind::ExtendsKeyword; + case "false"_hash: + return SyntaxKind::FalseKeyword; + case "finally"_hash: + return SyntaxKind::FinallyKeyword; + case "for"_hash: + return SyntaxKind::ForKeyword; + case "from"_hash: + return SyntaxKind::FromKeyword; + case "function"_hash: + return SyntaxKind::FunctionKeyword; + case "get"_hash: + return SyntaxKind::GetKeyword; + case "if"_hash: + return SyntaxKind::IfKeyword; + case "implements"_hash: + return SyntaxKind::ImplementsKeyword; + case "import"_hash: + return SyntaxKind::ImportKeyword; + case "in"_hash: + return SyntaxKind::InKeyword; + case "infer"_hash: + return SyntaxKind::InferKeyword; + case "instanceof"_hash: + return SyntaxKind::InstanceOfKeyword; + case "interface"_hash: + return SyntaxKind::InterfaceKeyword; + case "intrinsic"_hash: + return SyntaxKind::IntrinsicKeyword; + case "is"_hash: + return SyntaxKind::IsKeyword; + case "keyof"_hash: + return SyntaxKind::KeyOfKeyword; + case "let"_hash: + return SyntaxKind::LetKeyword; + case "module"_hash: + return SyntaxKind::ModuleKeyword; + case "namespace"_hash: + return SyntaxKind::NamespaceKeyword; + case "never"_hash: + return SyntaxKind::NeverKeyword; + case "new"_hash: + return SyntaxKind::NewKeyword; + case "null"_hash: + return SyntaxKind::NullKeyword; + case "number"_hash: + return SyntaxKind::NumberKeyword; + case "object"_hash: + return SyntaxKind::ObjectKeyword; + case "package"_hash: + return SyntaxKind::PackageKeyword; + case "private"_hash: + return SyntaxKind::PrivateKeyword; + case "protected"_hash: + return SyntaxKind::ProtectedKeyword; + case "public"_hash: + return SyntaxKind::PublicKeyword; + case "override"_hash: + return SyntaxKind::OverrideKeyword; + case "out"_hash: + return SyntaxKind::OutKeyword; + case "readonly"_hash: + return SyntaxKind::ReadonlyKeyword; + case "require"_hash: + return SyntaxKind::RequireKeyword; + case "global"_hash: + return SyntaxKind::GlobalKeyword; + case "return"_hash: + return SyntaxKind::ReturnKeyword; + case "set"_hash: + return SyntaxKind::SetKeyword; + case "static"_hash: + return SyntaxKind::StaticKeyword; + case "string"_hash: + return SyntaxKind::StringKeyword; + case "super"_hash: + return SyntaxKind::SuperKeyword; + case "switch"_hash: + return SyntaxKind::SwitchKeyword; + case "symbol"_hash: + return SyntaxKind::SymbolKeyword; + case "this"_hash: + return SyntaxKind::ThisKeyword; + case "throw"_hash: + return SyntaxKind::ThrowKeyword; + case "true"_hash: + return SyntaxKind::TrueKeyword; + case "try"_hash: + return SyntaxKind::TryKeyword; + case "type"_hash: + return SyntaxKind::TypeKeyword; + case "typeof"_hash: + return SyntaxKind::TypeOfKeyword; + case "undefined"_hash: + return SyntaxKind::UndefinedKeyword; + case "unique"_hash: + return SyntaxKind::UniqueKeyword; + case "unknown"_hash: + return SyntaxKind::UnknownKeyword; + case "var"_hash: + return SyntaxKind::VarKeyword; + case "void"_hash: + return SyntaxKind::VoidKeyword; + case "while"_hash: + return SyntaxKind::WhileKeyword; + case "with"_hash: + return SyntaxKind::WithKeyword; + case "yield"_hash: + return SyntaxKind::YieldKeyword; + case "async"_hash: + return SyntaxKind::AsyncKeyword; + case "await"_hash: + return SyntaxKind::AwaitKeyword; + case "of"_hash: + return SyntaxKind::OfKeyword; } + +// auto it = textToKeyword.find(tokenValue); +// if (it != textToKeyword.end()) { +// return token = it->second; +// } } } return token = SyntaxKind::Identifier; } - SyntaxKind Scanner::scanIdentifier(CharCode startCharacter, ScriptTarget languageVersion) { + SyntaxKind Scanner::scanIdentifier(const CharCode &startCharacter, ScriptTarget languageVersion) { + ZoneScoped; auto ch = startCharacter; if (isIdentifierStart(ch, languageVersion)) { pos += ch.length; @@ -1118,6 +1215,7 @@ namespace ts { } SyntaxKind Scanner::scan() { + ZoneScoped; startPos = pos; tokenFlags = TokenFlags::None; bool asteriskSeen = false; @@ -1132,7 +1230,7 @@ namespace ts { auto ch = charCodeAt(text, pos); // Special handling for shebang - if (ch.code == CharacterCodes::hash && pos == 0 && isShebangTrivia(text, pos)) { + if (pos == 0 && ch.code == CharacterCodes::hash && isShebangTrivia(text, pos)) { pos = scanShebangTrivia(text, pos); if (skipTrivia) { continue; @@ -1611,6 +1709,7 @@ namespace ts { } SyntaxKind Scanner::reScanGreaterToken() { + ZoneScoped; if (token == SyntaxKind::GreaterThanToken) { if (charCodeAt(text, pos).code == CharacterCodes::greaterThan) { if (charCodeAt(text, pos + 1).code == CharacterCodes::greaterThan) { @@ -1634,6 +1733,7 @@ namespace ts { } SyntaxKind Scanner::reScanSlashToken() { + ZoneScoped; if (token == SyntaxKind::SlashToken || token == SyntaxKind::SlashEqualsToken) { auto p = tokenPos + 1; auto inEscape = false; @@ -1684,6 +1784,7 @@ namespace ts { } SyntaxKind Scanner::reScanInvalidIdentifier() { + ZoneScoped; // Debug.assert(token == SyntaxKind::Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind::Unknown'."); pos = tokenPos = startPos; tokenFlags = 0; diff --git a/src/scanner.h b/src/scanner.h index 335876d..1077b02 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -16,8 +16,8 @@ namespace ts { string value; }; using ErrorCallback = function< - void(DiagnosticMessage - message, + void(const shared + &message, int length )>; @@ -412,7 +412,7 @@ namespace ts { SyntaxKind scan(); - SyntaxKind scanIdentifier(CharCode startCharacter, ScriptTarget languageVersion); + SyntaxKind scanIdentifier(const CharCode &startCharacter, ScriptTarget languageVersion); bool hasUnicodeEscape() { //todo @@ -457,7 +457,7 @@ namespace ts { } bool hasPrecedingJSDocComment() { - //todo + //JSDoc is not supported return false; } @@ -492,17 +492,17 @@ namespace ts { template T speculationHelper(const function &callback, bool isLookahead) { ZoneScoped; - auto savePos = pos; - auto saveStartPos = startPos; - auto saveTokenPos = tokenPos; - auto saveToken = token; - auto saveTokenValue = tokenValue; - auto saveTokenFlags = tokenFlags; - auto result = callback(); + const auto savePos = pos; + const auto saveStartPos = startPos; + const auto saveTokenPos = tokenPos; + const auto saveToken = token; + const auto saveTokenValue = tokenValue; + const auto saveTokenFlags = tokenFlags; + const auto result = callback(); // If our callback returned something 'falsy' or we're just looking ahead, // then unconditionally restore us to where we were. - if (!(bool)result || isLookahead) { + if (isLookahead || !(bool)result) { pos = savePos; startPos = saveStartPos; tokenPos = saveTokenPos; @@ -510,7 +510,7 @@ namespace ts { tokenValue = saveTokenValue; tokenFlags = saveTokenFlags; } - return result;; + return result; } bool hasPrecedingLineBreak() { @@ -621,7 +621,7 @@ namespace ts { bool isOctalDigit(const CharCode &code); - int error(const DiagnosticMessage &message, int errPos = -1, int length = -1); + int error(const shared &message, int errPos = -1, int length = -1); vector appendIfCommentDirective(vector &commentDirectives, const string &text, const regex &commentDirectiveRegEx, int lineStart); diff --git a/src/string.h b/src/string.h new file mode 100644 index 0000000..f98b113 --- /dev/null +++ b/src/string.h @@ -0,0 +1,13 @@ +/** + * A lighter string replacement. Since most nodes like Identifier/StringLiteral/NumberLiteral etc + * do not need std::string with all its features, it's enough if we either have a string_view (reference) + * to the SourceFile::text, or have a static string wrapper around const char *. + */ +namespace ts { + + struct simple_string { + const char *text; + int size; + }; + +} \ No newline at end of file diff --git a/src/tests/CmakeLists.txt b/src/tests/CmakeLists.txt index 529dd19..22a3249 100644 --- a/src/tests/CmakeLists.txt +++ b/src/tests/CmakeLists.txt @@ -12,8 +12,6 @@ FetchContent_Declare( #set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) -include_directories(../../libs) - #include_directories(${}) diff --git a/src/tests/test_parser.cpp b/src/tests/test_parser.cpp index 31bfd16..85cbcf5 100644 --- a/src/tests/test_parser.cpp +++ b/src/tests/test_parser.cpp @@ -2,11 +2,14 @@ #include #include "../parser2.h" +#include using namespace ts; -TEST(parser, bench) { +TEST(parser, single) { Parser parser; + + auto code = "const i = 123;"; /** ConstKeyword WhitespaceTrivia @@ -19,11 +22,20 @@ SemicolonToken EndOfFileToken */ + auto result = parser.parseSourceFile("app.ts", code, ts::types::ScriptTarget::Latest, false, ScriptKind::TS, {}); + debug("done"); +} + +TEST(parser, bench) { + Parser parser; string code; + for (int i = 0; i <20000; i++) { code += string("const i").append(to_string(i)).append(" = 123;"); } + usleep(100'000); + auto start = std::chrono::high_resolution_clock::now(); auto i = 0; @@ -32,5 +44,7 @@ EndOfFileToken // auto sourceFile = parser.createSourceFile("app.ts", ts::types::ScriptTarget::Latest, ScriptKind::TS, false, make_shared(), make_shared(), 0, [](auto s) {}); // } std::chrono::duration took = std::chrono::high_resolution_clock::now() - start; - debug("parse %d bytes took %fms", code.size(), took.count()); + fmt::print("parse {} bytes took {}ms", code.size(), took.count()); + + usleep(100'000); } \ No newline at end of file diff --git a/src/tests/test_scanner.cpp b/src/tests/test_scanner.cpp index bee5951..6aa82fb 100644 --- a/src/tests/test_scanner.cpp +++ b/src/tests/test_scanner.cpp @@ -29,7 +29,7 @@ TEST(scanner, basisc) { scanner.setOnError(nullopt); } std::chrono::duration took = std::chrono::high_resolution_clock::now() - start; - debug("scan %d took %fms", i, took.count()); + debug("scan {} took {}ms", i, took.count()); // std::cout << enum_name(scanner.scan()) << "\n"; // std::cout << enum_name(scanner.scan()) << "\n"; diff --git a/src/types.cpp b/src/types.cpp index a9d21e1..89006d2 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -12,4 +12,5 @@ namespace ts { // BaseUnion::BaseUnion() { // node = make_shared(); // } + types::DiagnosticMessage::DiagnosticMessage(int code, types::DiagnosticCategory category, const string_view &key, const string_view &message, bool reportsUnnecessary, bool reportsDeprecated, bool elidedInCompatabilityPyramid): code(code), category(category), key(key), message(message), reportsUnnecessary(reportsUnnecessary), reportsDeprecated(reportsDeprecated), elidedInCompatabilityPyramid(elidedInCompatabilityPyramid) {} } \ No newline at end of file diff --git a/src/types.h b/src/types.h index 9186d64..01db06f 100644 --- a/src/types.h +++ b/src/types.h @@ -10,6 +10,9 @@ #include #include #include "core.h" +#include +#include +#include "magic_enum.hpp" namespace ts { struct SourceFile; @@ -32,8 +35,8 @@ namespace ts::types { /* @internal */ enum class Comparison { - LessThan = -1, - EqualTo = 0, + LessThan = -1, + EqualTo = 0, GreaterThan = 1 }; @@ -97,12 +100,14 @@ namespace ts::types { struct DiagnosticMessage { int code; DiagnosticCategory category; - string key; - string message; + string_view key; + string_view message; bool reportsUnnecessary; - bool reportsDeprecated; /* @internal */ bool elidedInCompatabilityPyramid; + bool reportsDeprecated; + + DiagnosticMessage(int code, DiagnosticCategory category, const string_view &key, const string_view &message, bool reportsUnnecessary, bool elidedInCompatabilityPyramid, bool reportsDeprecated); }; struct DiagnosticMessageChain { @@ -116,9 +121,9 @@ namespace ts::types { DiagnosticCategory category; int code; SourceFile *file; - int start = - 1; //-1 = undefined - int length = - 1; //-1 = undefined - string messageText; + int start = -1; //-1 = undefined + int length = -1; //-1 = undefined + string_view messageText; DiagnosticMessageChain *messageChain = nullptr; }; @@ -1020,7 +1025,7 @@ namespace ts::types { PropertyExcludes = NodeExcludes | ContainsLexicalThis | ContainsLexicalSuper, ClassExcludes = NodeExcludes | ContainsTypeScriptClassSyntax | ContainsComputedPropertyName, ModuleExcludes = NodeExcludes | ContainsTypeScriptClassSyntax | ContainsLexicalThis | ContainsLexicalSuper | ContainsBlockScopedBinding | ContainsHoistedDeclarationOrCompletion | ContainsPossibleTopLevelAwait, - TypeExcludes = ~ ContainsTypeScript, + TypeExcludes = ~ContainsTypeScript, ObjectLiteralExcludes = NodeExcludes | ContainsTypeScriptClassSyntax | ContainsComputedPropertyName | ContainsObjectRestOrSpread, ArrayLiteralOrCallOrNewExcludes = NodeExcludes | ContainsRestOrSpread, VariableDeclarationListExcludes = NodeExcludes | ContainsBindingPattern | ContainsObjectRestOrSpread, @@ -1078,7 +1083,6 @@ namespace ts { return kinds; } - struct Decorator; struct Modifier; @@ -1181,6 +1185,14 @@ namespace ts { bool isMissingList = false; //replaces `MissingList extends NodeArray {bool isMissingList;}` /* @internal */ int transformFlags = 0; // Flags for transforms, possibly undefined + NodeArray() {} + + NodeArray(const vector> &list, bool hasTrailingComma = false): list(list), hasTrailingComma(hasTrailingComma) {} + + NodeArray(const shared &item) { + list.push_back(item); + } + int length() { return list.size(); } @@ -1204,7 +1216,8 @@ namespace ts { } vector> slice(int start, int end = 0) { - return slice>(list, start, end); + return slice < shared> + (list, start, end); } }; @@ -1301,14 +1314,15 @@ namespace ts { // /* @internal */ contextualType?: Type; // Used to temporarily assign a contextual type during overload resolution // /* @internal */ inferenceContext?: InferenceContext; // Inference context for contextual type - virtual ~Node() {} + virtual ~Node() { + } bool hasParent() { return &parent != this; } Node &getParent() { - if (! hasParent()) throw std::runtime_error("Node has no parent set"); + if (!hasParent()) throw std::runtime_error("Node has no parent set"); return parent; } @@ -1333,7 +1347,7 @@ namespace ts { template T &to() { if (T::KIND == SyntaxKind::Unknown) throw runtime_error("Passed Node type has unknown kind."); - if (kind != T::KIND) throw std::runtime_error(format("Can not convert Node, from kind %d to %d", kind, T::KIND)); + if (kind != T::KIND) throw std::runtime_error(fmt::format("Can not convert Node, from kind {} to {}", kind, T::KIND)); return *reinterpret_cast(this); } }; @@ -2855,4 +2869,24 @@ namespace ts { // /* @internal */ endFlowNode?: FlowNode; }; -} \ No newline at end of file +} + +template<> +struct fmt::formatter { +// // Presentation format: 'f' - fixed, 'e' - exponential. +// char presentation = 'f'; + + // Parses format specifications of the form ['f' | 'e']. + constexpr auto parse(format_parse_context &ctx)->decltype(ctx.begin()) { + return ctx.begin(); + } + + // Formats the point p using the parsed format specification (presentation) + // stored in this formatter. + template + auto format(const ts::types::SyntaxKind &p, FormatContext &ctx)->decltype(ctx.out()) { + fmt::format_to(ctx.out(), magic_enum::enum_name(p)); + // ctx.out() is an output iterator to write to. +// return magic_enum::enum_name(p); + } +}; diff --git a/src/utilities.cpp b/src/utilities.cpp index 45e7d12..7234c86 100644 --- a/src/utilities.cpp +++ b/src/utilities.cpp @@ -4,6 +4,7 @@ #include "types.h" #include "utilities.h" #include "core.h" +#include namespace ts { LanguageVariant getLanguageVariant(ScriptKind scriptKind) { @@ -258,17 +259,17 @@ namespace ts { //unordered_map localizedDiagnosticMessages{}; - string getLocaleSpecificMessage(DiagnosticMessage message) { + string_view getLocaleSpecificMessage(const shared &message) { // if (has(localizedDiagnosticMessages, message.key)) { // return localizedDiagnosticMessages[message.key]; // } - return message.message; + return message->message; } using DiagnosticArg = string; string formatStringFromArg(string &text, int i, string &v) { - return replaceAll(text, format("{%d}", i), v); + return replaceAll(text, fmt::format("{}", i), v); // return text.replace(/{(\d+)}/g, (_match, index: string) => "" + Debug.checkDefined(args[+index + baseIndex])); } @@ -276,7 +277,7 @@ namespace ts { return v; //currently only string supported } - DiagnosticWithDetachedLocation createDetachedDiagnostic(string fileName, int start, int length, DiagnosticMessage message, vector textArg) { + DiagnosticWithDetachedLocation createDetachedDiagnostic(string fileName, int start, int length, const shared &message, vector textArg) { // assertDiagnosticLocation(/*file*/ undefined, start, length); auto text = getLocaleSpecificMessage(message); @@ -296,10 +297,10 @@ namespace ts { { { .messageText = text, - .category = message.category, - .code = message.code, + .category = message->category, + .code = message->code, }, - .reportsUnnecessary = message.reportsUnnecessary, + .reportsUnnecessary = message->reportsUnnecessary, }, .fileName = fileName, .start = start, @@ -417,7 +418,7 @@ namespace ts { case SyntaxKind::PartiallyEmittedExpression: return node->to().expression; } - throw runtime_error(format("No expression found in type %d", (int) node->kind)); + throw runtime_error(fmt::format("No expression found in type {}", (int) node->kind)); } shared skipOuterExpressions(shared node, int kinds) { diff --git a/src/utilities.h b/src/utilities.h index fb5998c..2b6450e 100644 --- a/src/utilities.h +++ b/src/utilities.h @@ -294,7 +294,7 @@ namespace ts { //unordered_map localizedDiagnosticMessages{}; - string getLocaleSpecificMessage(DiagnosticMessage message); + string_view getLocaleSpecificMessage(const shared &message); using DiagnosticArg = string; @@ -302,7 +302,7 @@ namespace ts { string DiagnosticArgToString(DiagnosticArg &v) ; - DiagnosticWithDetachedLocation createDetachedDiagnostic(string fileName, int start, int length, DiagnosticMessage message, vector textArg = {}); + DiagnosticWithDetachedLocation createDetachedDiagnostic(string fileName, int start, int length, const shared &message, vector textArg = {}); DiagnosticWithDetachedLocation &addRelatedInfo(DiagnosticWithDetachedLocation &diagnostic, vector relatedInformation) ;