benchmark in readme

CMakeLists.txt

@@ -31,8 +31,7 @@ include_directories(libs/magic_enum)
 add_subdirectory(src)
 
 add_executable(typescript_main main.cpp)
+target_link_libraries(typescript_main typescript)
 
-target_link_libraries(
-        typescript_main
-        typescript
-)
+add_executable(bench bench.cpp)
+target_link_libraries(bench typescript)

README.md

@@ -11,11 +11,12 @@ High-performance TypeScript compiler.
 - Type information in other languages
 - (optional) transpiling to JavaScript
 - (optional) RTTI in JavaScript
+- (optional) type profiler
 
-The goal is to make TypeScript type checking as fast as possible and provide alongside with it a native TS library for other languages, so they can use TypeScript type information
+The goal is to make TypeScript type checking as fast as possible and provide alongside with it a native library for other languages, so they can use TypeScript type information
 without the need for a JavaScript engine for all sorts of use cases like JSON-Schema replacement, ORM DSL, encoding information (like Protocol Buffers schema) and more.
 
-The goal is not to be a drop-in replacement for the entire official TypeScript compiler (tsc). TypeScript just supports so much that is not always necessary. 
+The goal is not to be a drop-in replacement for the entire official TypeScript compiler (tsc). TypeScript supports so much that is not always necessary. 
 We focus on the more strict TS part which means TypeRunner won't support JSDoc and a lot of compiler options.  
 
 ## Status
@@ -24,12 +25,81 @@ The source code in the initial version is really only a proof of concept. It con
 The approach is a TS-to-bytecode compiler and then run the bytecode in a custom virtual machine.
 The data show that this approach can lead to a hundred- to several-thousand-fold improvement in speed.
 
--- todo show data as graphs, maybe even make auto-generated --
-
 ![TypeRunner Debugger](./docs/typerunner-debugger.png)
 
 Once the project gets funding through the community, the development will continue.
 
+## Performance
+
+`TypeRunner cold` means the file was seen for the first time and has to be compiled to bytecode first.
+`TypeRunner warm` means the bytecode could be directly executed because it was cached. Generally it is assumed
+that only a few files change, for example if you have a project with 100 projects and edit one and then rerun 
+type checking, then only the changed one has the slower `cold` timing. Note that compilation has not yet been optimised 
+(it still uses a slow memory allocator which can be improvement by a tenfold). 
+
+Note that `tsc` numbers are after 10 iterations (the JavaScript engine V8 JIT optimises it early already), which somewhat
+leads to a wrong conclusion. Only 1 iteration is 10x slower and a cold `tsc` start even slower because of the initial bootstrap
+delay of several hundred milliseconds. So you can generally assume that `tsc` is slower than the numbers shown below.
+
+### Basic variables
+
+```typescript
+const v1: string = "abc";
+const v2: number = 123;
+```
+
+```
+TypeScript tsc:  1.1ms
+TypeRunner cold: 0.008374167ms (131x faster)
+TypeRunner warm: 0.000104375ms (10,576x faster)
+```
+
+### Generic function
+
+```typescript
+function doIt<T extends number>(v: T) {
+}
+const a = doIt<number>;
+a(23);
+```
+
+```
+TypeScript tsc:  1.4ms
+TypeRunner cold: 0.014966250ms (93x faster)
+TypeRunner warm: 0.000181875ms (7,697x faster)
+```
+
+### Object literal type
+
+```typescript
+type Person = {name: string, age: number}
+
+const a: Person = {name: 'Peter', age: 52};
+const b: Person = {name: 'Peter', age: '52'};
+```
+
+```
+TypeScript tsc:  1.9ms
+TypeRunner cold: 0.021316125ms (89x faster)
+TypeRunner warm: 0.001111333ms (1,709x faster)
+```
+
+### Complex type
+
+```
+```typescript
+type StringToNum<T extends string, A extends 0[] = []> = `${A['length']}` extends T ? A['length'] : StringToNum<T, [...A, 0]>;
+const var1: StringToNum<'999'> = 999;
+```
+
+```
+TypeScript tsc:  350.2ms
+TypeRunner cold:   0.862534792ms (406x faster)
+TypeRunner warm:   0.839308334ms (417x faster)
+```
+
+```
+
 ## Development
 
 TypeRunner is written in modern C++ with cmake, doctest, imgui, tracy, fmt. To work on this project first clone the repository:

bench.cpp

@@ -0,0 +1,61 @@
+#include <iostream>
+#include <memory>
+#include <unistd.h>
+
+#include "./src/core.h"
+#include "./src/fs.h"
+#include "./src/parser2.h"
+#include "./src/checker/vm2.h"
+#include "./src/checker/module2.h"
+#include "./src/checker/debug.h"
+#include "./src/checker/compiler.h"
+
+using namespace tr;
+
+void compileAndRun(const string &code, const string &fileName) {
+    ZoneScoped;
+    auto iterations = 1000;
+    auto cold = benchRun(iterations, [&] {
+        checker::Compiler compiler;
+        Parser parser;
+        auto result = parser.parseSourceFile(fileName, code, types::ScriptTarget::Latest, false, ScriptKind::TS, {});
+        auto program = compiler.compileSourceFile(result);
+        auto bin = program.build();
+        auto module = make_shared<vm2::Module>(bin, fileName, code);
+        vm2::run(module);
+    });
+
+    checker::Compiler compiler;
+    Parser parser;
+    auto result = parser.parseSourceFile(fileName, code, types::ScriptTarget::Latest, false, ScriptKind::TS, {});
+    auto program = compiler.compileSourceFile(result);
+    auto bin = program.build();
+    auto module = make_shared<vm2::Module>(bin, fileName, code);
+    auto warm = benchRun(iterations, [&] {
+        module->clear();
+        vm2::run(module);
+    });
+
+    std::cout << fmt::format("typerunner: {} iterations (it): cold {:.9f}ms/it, warm {:.9f}ms/it\n", iterations, cold.count() / iterations, warm.count() / iterations);
+}
+
+int main(int argc, char *argv[]) {
+    ZoneScoped;
+    std::string file;
+    auto cwd = std::filesystem::current_path();
+
+    if (argc > 1) {
+        file = cwd.string() + "/" + argv[1];
+    } else {
+        file = cwd.string() + "/../tests/basic1.ts";
+    }
+
+    if (!fileExists(file)) {
+        std::cout << "File not found " << file << "\n";
+        return 4;
+    }
+    auto code = fileRead(file);
+    auto relative = std::filesystem::relative(file, cwd);
+    compileAndRun(code, relative.string());
+    return 0;
+}

main.cpp

@@ -10,8 +10,7 @@
 #include "./src/checker/debug.h"
 #include "./src/checker/compiler.h"
 
-using namespace ts;
-
+using namespace tr;
 
 void run(const string &bytecode, const string &code, const string &fileName) {
     ZoneScoped;
@@ -28,7 +27,7 @@ void compileAndRun(const string &code, const string &file, const string &fileNam
     auto buffer = fileRead(file);
     checker::Compiler compiler;
     Parser parser;
-    auto result = parser.parseSourceFile(file, buffer, ts::types::ScriptTarget::Latest, false, ScriptKind::TS, {});
+    auto result = parser.parseSourceFile(file, buffer, types::ScriptTarget::Latest, false, ScriptKind::TS, {});
     auto program = compiler.compileSourceFile(result);
     auto bin = program.build();
     fileWrite(bytecodePath, bin);
@@ -41,13 +40,19 @@ void compileAndRun(const string &code, const string &file, const string &fileNam
 
 int main(int argc, char *argv[]) {
     ZoneScoped;
-    std::string file = "/Users/marc/bude/TypeRunner/tests/basic1.ts";
+    std::string file;
     auto cwd = std::filesystem::current_path();
 
     if (argc > 1) {
         file = cwd.string() + "/" + argv[1];
+    } else {
+        file = cwd.string() + "/../tests/basic1.ts";
     }
 
+    if (!fileExists(file)) {
+        std::cout << "File not found " << file << "\n";
+        return 4;
+    }
     auto code = fileRead(file);
     auto bytecode = file + ".tsb";
     auto relative = std::filesystem::relative(file, cwd);
@@ -57,6 +62,5 @@ int main(int argc, char *argv[]) {
     } else {
         compileAndRun(code, file, relative.string());
     }
-    usleep(100000);
     return 0;
 }

package.json

@@ -10,7 +10,7 @@
     "test": "tests"
   },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "bench": "ts-node tests/bench.ts"
   },
   "repository": {
     "type": "git",

src/checker/vm2.cpp

@@ -1021,7 +1021,7 @@ namespace tr::vm2 {
                 case OP::Assign: {
                     auto rvalue = pop();
                     auto lvalue = pop();
-                    debug("assign {} = {}", stringify(rvalue), stringify(lvalue));
+                    //debug("assign {} = {}", stringify(rvalue), stringify(lvalue));
                     if (!extends(lvalue, rvalue)) {
 //                        auto error = stack.errorMessage();
 //                        error.ip = ip;

tests/bench.ts

@@ -1,11 +1,20 @@
 import {CompilerOptions, createCompilerHost, createProgram, getPreEmitDiagnostics} from "typescript";
+import {readFileSync} from "fs";
+import {execSync} from "child_process";
 
-const code = `
-    function doIt(): string {
-        return 1;
-    }
-    doIt();
-`;
+// const code = `
+//     function doIt(): string {
+//         return 1;
+//     }
+//     doIt();
+// `;
+const file = process.argv[2];
+const code = readFileSync(file).toString('utf8');
+
+console.log('file', file);
+console.log('code:');
+console.log(code);
+console.log();
 
 const options: CompilerOptions = {
     strict: true,
@@ -22,7 +31,6 @@ host.writeFile = () => {
 }
 
 const program = createProgram(['app.ts'], options, host);
-console.log(getPreEmitDiagnostics(program));
 
 const iterations = 10;
 const start = Date.now();
@@ -31,4 +39,6 @@ for (let i = 0; i < iterations; i++) {
     const diagnostics = getPreEmitDiagnostics(program);
 }
 const took = Date.now() - start;
-console.log(iterations, 'iterations took', took, 'ms.', took/iterations, 'ms/op');
+console.log('tsc: ', iterations, 'iterations took', took, 'ms.', took / iterations, 'ms/op');
+
+execSync(__dirname + '/../cmake-build-release/bench ' + file, {stdio: 'inherit'});

tests/function1.ts

@@ -0,0 +1,4 @@
+function doIt<T extends number>(v: T) {
+}
+const a = doIt<number>;
+a(23);