emscripten support

add http transport for emscripten
add example for git commit
fix example for git add
add synchronous http worker for nodejs
add web http support using sync XmlHttpRequest
add nodejs and webworker examples for interacting with libgit2.js
depends on mmap offset fix in emscripten:
emscripten-core/emscripten#10095

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "string_view": "c"
+    }
+}

emscripten/.gitignore

@@ -0,0 +1,9 @@
+CMakeFiles
+deps
+examples
+src
+cmake*
+CMake*
+libgit2.*
+Makefile
+node_modules

emscripten/README.md

@@ -0,0 +1,12 @@
+# Building libgit2 with emscripten
+
+Using [emscripten](https://emscripten.org) you may compile libgit2 to webassembly and run it in nodejs or in a web browser.
+
+The script in [build.sh](build.sh) shows how to configure and build, and you'll find the resulting lg2.js/lg2.wasm under the generated `examples` folder.
+
+An example of interacting with libgit2 from nodejs can be found in [example_node.js](example_node.js).
+
+An example for the browser (using webworkers) can be found in [example_webworker.js](example_webworker.js). You can start a webserver for this by running the [webserverwithgithubproxy.js](webserverwithgithubproxy.js) script, which will launch a http server at http://localhost:5000 with a proxy to github. Proxy instead of direct calls is needed because of CORS restrictions in a browser environment.
+
+This work depends on a patch for the mmap functionality in emscripten which can be found here: https://github.com/emscripten-core/emscripten/pull/10095
+You can copy the modified [library_syscall.js](https://github.com/emscripten-core/emscripten/blob/aaec9dba785b0d5245eae38f5e1ad9e1783d7205/src/library_syscall.js) over the one found in your emscripten installation ( `upstream/emscripten/src/library_syscall.js` ).

emscripten/build.sh

@@ -0,0 +1,3 @@
+emcmake cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="--post-js $(pwd)/post.js -s \"EXTRA_EXPORTED_RUNTIME_METHODS=['FS','callMain']\" -s INVOKE_RUN=0 -s ALLOW_MEMORY_GROWTH=1" -DREGEX_BACKEND=regcomp -DSONAME=OFF -DUSE_HTTPS=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_CLAR=OFF -DUSE_SSH=OFF -DBUILD_EXAMPLES=ON ..
+emcmake cmake -build .
+emmake make

emscripten/example_node.js

@@ -0,0 +1,50 @@
+const lg = require('./examples/lg2.js');
+
+lg.onRuntimeInitialized = () => {
+    const FS = lg.FS;
+    const MEMFS = FS.filesystems.MEMFS;
+
+    FS.mkdir('/working');
+    FS.mount(MEMFS, { }, '/working');
+    FS.chdir('/working');    
+
+    FS.writeFile('/home/web_user/.gitconfig', '[user]\n' +
+                'name = Test User\n' +
+                'email = [email protected]');
+
+    // clone a repository from github
+    lg.callMain(['clone','https://github.com/torch2424/made-with-webassembly.git','clonedtest']);
+
+    FS.chdir('clonedtest');
+    console.log(FS.readdir('.'));
+    lg.callMain(['log']);
+
+    FS.chdir('..');
+
+    // create an empty git repository and create some commits
+    lg.callMain(['init','testrepo']);
+    FS.chdir('testrepo');
+    FS.writeFile('test.txt', 'hello');
+
+    lg.callMain(['add', '--verbose', 'test.txt']);
+    lg.callMain(['commit','-m','test 123']);
+
+    lg.callMain(['log']);
+    lg.callMain(['status']);
+
+
+    lg.callMain(['status']);
+
+    FS.writeFile('test.txt', 'second revision');
+
+    lg.callMain(['add', 'test.txt']);
+
+    lg.callMain(['status']);
+    lg.callMain(['commit','-m','test again']);
+
+    lg.callMain(['status']);
+
+    lg.callMain(['log']);
+
+    FS.chdir('..');
+};

emscripten/example_webworker.js

@@ -0,0 +1,49 @@
+importScripts('lg2.js');
+
+Module.onRuntimeInitialized = () => {
+    const lg = Module;
+
+    FS.mkdir('/working');
+    FS.mount(MEMFS, { }, '/working');
+    FS.chdir('/working');    
+
+    FS.writeFile('/home/web_user/.gitconfig', '[user]\n' +
+                'name = Test User\n' +
+                'email = [email protected]');
+
+    // create an empty git repository and create some commits
+
+    lg.callMain(['init','testrepo']);
+    FS.chdir('testrepo');
+    FS.writeFile('test.txt', 'hello');
+
+    lg.callMain(['add', '--verbose', 'test.txt']);
+    lg.callMain(['commit','-m','test 123']);
+
+    lg.callMain(['log']);
+    lg.callMain(['status']);
+
+
+    lg.callMain(['status']);
+
+    FS.writeFile('test.txt', 'second revision');
+
+    lg.callMain(['add', 'test.txt']);
+
+    lg.callMain(['status']);
+    lg.callMain(['commit','-m','test again']);
+
+    lg.callMain(['status']);
+
+    lg.callMain(['log']);
+
+    FS.chdir('..');
+
+    // clone a repository from github
+    lg.callMain(['clone','http://localhost:5000/torch2424/made-with-webassembly.git','clonedtest']);
+
+    FS.chdir('clonedtest');
+    console.log(FS.readdir('.'));
+    lg.callMain(['log']);
+
+};

emscripten/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+    <head>
+
+    </head>
+    <body>
+        Open the webconsole to see the output of the <a href="example_webworker.js" target="_blank">example_webworker.js</a> script
+        <script>
+            new Worker('example_webworker.js');
+        </script>
+    </body>
+</html>

emscripten/node-sync-fetch.js

@@ -0,0 +1,55 @@
+const { Worker, isMainThread, workerData } = require('worker_threads');
+
+if (isMainThread) {
+    const statusArray = new Int32Array(new SharedArrayBuffer(4));
+    Atomics.store(statusArray, 0, 0);
+
+    const resultBuffer = new SharedArrayBuffer(65536);
+    const resultArray = new Uint8Array(resultBuffer);
+    const worker = new Worker(__filename, {
+        workerData: {
+            statusArray: statusArray,
+            resultArray: resultArray
+        }
+    }); 
+
+    while(true) {
+        Atomics.wait(statusArray, 0, 0);
+        const length = statusArray[0];
+        if(length === -1 ) {
+            console.log('thats all');
+            break;
+        }
+
+        Atomics.store(statusArray, 0, 0);
+        Atomics.notify(statusArray, 0, 1);
+        // console.log(new TextDecoder("utf-8").decode(resultArray.slice(0,length)));
+        console.log(length);
+    }
+} else {
+    const req = require('https').request('https://petersalomonsen.com',
+        (res) => {
+        res.on('data', chunk => {
+            if(workerData.statusArray[0] !== 0) {
+                Atomics.wait(workerData.statusArray, 0, workerData.statusArray[0]);
+            }
+            for(let n=0;n<chunk.length;n++) {
+                workerData.resultArray[n] = chunk[n];
+            }
+            // console.log('chunk size ', chunk.length);
+            Atomics.store(workerData.statusArray, 0, chunk.length);
+            Atomics.notify(workerData.statusArray, 0, 1);
+        });
+        res.on('end', () => {
+            Atomics.store(workerData.statusArray, 0, -1);
+            Atomics.notify(workerData.statusArray, 0, 1);
+        });
+    });        
+    // if(workerData.method === 'POST') {   
+        // console.log(workerData.resultArray); 
+        console.log(workerData.resultArray.slice(0, 4));        
+         req.write(Buffer.from(workerData.resultArray.slice(0, 4)));
+    // }
+
+    req.end();
+}

emscripten/post.js

@@ -0,0 +1,153 @@
+const emscriptenhttpconnections = {};
+let httpConnectionNo = 0;
+
+const nodePermissions = FS.nodePermissions;
+
+FS.nodePermissions = function(node, perms) { 
+    if(node.mode & 0o100000) {
+        /* 
+            * Emscripten doesn't support the sticky bit, while libgit2 sets this on some files.
+            * grant permission if sticky bit is set
+            */        
+        return 0;
+    } else {
+        return nodePermissions(node, perms);
+    }
+};
+
+if(ENVIRONMENT_IS_WORKER) {
+    Object.assign(Module, {
+        emscriptenhttpconnect: function(url, buffersize, method, headers) {
+            if(!method) {
+                method = 'GET';
+            }
+
+            const xhr = new XMLHttpRequest();
+            xhr.open(method, url, false);
+            xhr.responseType = 'arraybuffer';
+
+            if (headers) {
+                Object.keys(headers).forEach(header => xhr.setRequestHeader(header, headers[header]));
+            }
+
+            emscriptenhttpconnections[httpConnectionNo] = {
+                xhr: xhr,
+                resultbufferpointer: 0,
+                buffersize: buffersize
+            };
+
+            if(method === 'GET') {
+                xhr.send();
+            }
+
+            return httpConnectionNo++;
+        },
+        emscriptenhttpwrite: function(connectionNo, buffer, length) {
+            const xhr = emscriptenhttpconnections[connectionNo].xhr;
+            xhr.send(new Uint8Array(Module.HEAPU8.buffer,buffer,length));
+        },
+        emscriptenhttpread: function(connectionNo, buffer, buffersize) { 
+            const connection = emscriptenhttpconnections[connectionNo];
+            let bytes_read = connection.xhr.response.byteLength - connection.resultbufferpointer;
+            if (bytes_read > buffersize) {
+                bytes_read = buffersize;
+            }
+            const responseChunk = new Uint8Array(connection.xhr.response, connection.resultbufferpointer, bytes_read);
+            writeArrayToMemory(responseChunk, buffer);
+            connection.resultbufferpointer += bytes_read;
+            return bytes_read;
+        },
+        emscriptenhttpfree: function(connectionNo) {
+            delete emscriptenhttpconnections[connectionNo];
+        }
+    });
+} else if(ENVIRONMENT_IS_NODE) {
+    const { Worker } = require('worker_threads');
+
+    Object.assign(Module, {
+        emscriptenhttpconnect: function(url, buffersize, method, headers) {
+            const statusArray = new Int32Array(new SharedArrayBuffer(4));
+            Atomics.store(statusArray, 0, method === 'POST' ? -1 : 0);
+
+            const resultBuffer = new SharedArrayBuffer(buffersize);
+            const resultArray = new Uint8Array(resultBuffer);
+            const workerData =  {
+                    statusArray: statusArray,
+                    resultArray: resultArray,
+                    url: url,
+                    method: method ? method: 'GET',
+                    headers: headers
+            };  
+
+            new Worker('(' + (function requestWorker() {
+                const { workerData } = require('worker_threads');
+                const req = require('https').request(workerData.url, {
+                    headers: workerData.headers,
+                    method: workerData.method
+                }, (res) => {
+                    res.on('data', chunk => {
+                        const previousStatus = workerData.statusArray[0];
+                        if(previousStatus !== 0) {
+                            Atomics.wait(workerData.statusArray, 0, previousStatus);
+                        }                    
+                        workerData.resultArray.set(chunk);                    
+                        Atomics.store(workerData.statusArray, 0, chunk.length);
+                        Atomics.notify(workerData.statusArray, 0, 1);
+                    });
+                });        
+
+                if(workerData.method === 'POST') {
+                    while(workerData.statusArray[0] !== 0) {
+                        Atomics.wait(workerData.statusArray, 0, -1);
+                        const length = workerData.statusArray[0];
+                        if(length === 0) {
+                            break;
+                        }
+                        req.write(Buffer.from(workerData.resultArray.slice(0, length)));
+                        Atomics.store(workerData.statusArray, 0, -1);
+                        Atomics.notify(workerData.statusArray, 0, 1);
+                    }
+                }
+
+                req.end();
+            }).toString()+')()' , {
+                eval: true,
+                workerData: workerData
+            }); 
+            emscriptenhttpconnections[httpConnectionNo] = workerData;
+            console.log('connected with method', workerData.method, 'to', workerData.url);
+            return httpConnectionNo++;
+        },
+        emscriptenhttpwrite: function(connectionNo, buffer, length) {
+            const connection = emscriptenhttpconnections[connectionNo];
+            connection.resultArray.set(new Uint8Array(Module.HEAPU8.buffer,buffer,length));
+            Atomics.store(connection.statusArray, 0, length);
+            Atomics.notify(connection.statusArray, 0, 1);
+            // Wait for write to finish
+            Atomics.wait(connection.statusArray, 0, length);
+        },
+        emscriptenhttpread: function(connectionNo, buffer) { 
+            const connection = emscriptenhttpconnections[connectionNo];
+
+            if(connection.statusArray[0] === -1 && connection.method === 'POST') {
+                // Stop writing
+                Atomics.store(connection.statusArray, 0, 0);
+                Atomics.notify(connection.statusArray, 0, 1);
+            }
+            Atomics.wait(connection.statusArray, 0, 0);
+            const bytes_read = connection.statusArray[0];
+
+            writeArrayToMemory(connection.resultArray.slice(0, bytes_read), buffer);
+
+            //console.log('read with connectionNo', connectionNo, 'length', bytes_read, 'content',
+            //        new TextDecoder('utf-8').decode(connection.resultArray.slice(0, bytes_read)));
+            Atomics.store(connection.statusArray, 0, 0);
+            Atomics.notify(connection.statusArray, 0, 1);
+
+            return bytes_read;
+        },
+        emscriptenhttpfree: function(connectionNo) {
+            delete emscriptenhttpconnections[connectionNo];
+        }
+    });
+}